2026-03-20T11:17:30.273 INFO:root:teuthology version: 1.2.4.dev6+g1c580df7a 2026-03-20T11:17:30.279 DEBUG:teuthology.report:Pushing job info to http://localhost:8080 2026-03-20T11:17:30.306 INFO:teuthology.run:Config: archive_path: /archive/kyr-2026-03-20_10:58:43-rgw-tentacle-none-default-vps/2068 branch: tentacle description: rgw/multisite/{clusters frontend/beast ignore-pg-availability notify omap_limits overrides realms/two-zonegroup supported-random-distro$/{centos_latest} tasks/test_multi} email: null first_in_suite: false flavor: default job_id: '2068' last_in_suite: false machine_type: vps name: kyr-2026-03-20_10:58:43-rgw-tentacle-none-default-vps no_nested_subset: false os_type: centos os_version: 9.stream overrides: admin_socket: branch: tentacle ansible.cephlab: branch: main repo: https://github.com/kshtsk/ceph-cm-ansible.git skip_tags: nagios,monitoring-scripts,hostname,pubkeys,zap,sudoers,kerberos,ntp-client,resolvconf,cpan,nfs vars: logical_volumes: lv_1: scratch_dev: true size: 25%VG vg: vg_nvme lv_2: scratch_dev: true size: 25%VG vg: vg_nvme lv_3: scratch_dev: true size: 25%VG vg: vg_nvme lv_4: scratch_dev: true size: 25%VG vg: vg_nvme timezone: UTC volume_groups: vg_nvme: pvs: /dev/vdb,/dev/vdc,/dev/vdd,/dev/vde ceph: conf: client: debug rgw: 20 debug rgw lifecycle: 20 debug rgw notification: 20 debug rgw sync: 20 rgw crypt require ssl: false rgw crypt s3 kms backend: testing rgw crypt s3 kms encryption keys: testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= rgw curl low speed time: 300 rgw data log num shards: 4 rgw data sync poll interval: 5 rgw md log max shards: 4 rgw meta sync poll interval: 5 rgw s3 auth use sts: true rgw sts key: abcdefghijklmnoq rgw sync data inject err probability: 0 rgw sync log trim interval: 0 rgw sync meta inject err probability: 0 rgw sync obj etag verify: true setgroup: ceph setuser: ceph client.0: rgw data notify interval msec: 0 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd mclock iops capacity threshold hdd: 49000 rocksdb delete range threshold: 0 osd.0: osd_max_omap_entries_per_request: 10 osd.1: osd_max_omap_entries_per_request: 1000 osd.2: osd_max_omap_entries_per_request: 10000 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(PG_AVAILABILITY\) - \(PG_DEGRADED\) - \(POOL_APP_NOT_ENABLED\) - not have an application enabled sha1: 70f8415b300f041766fa27faf7d5472699e32388 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} cephadm: cephadm_binary_url: https://download.ceph.com/rpm-20.2.0/el9/noarch/cephadm install: ceph: flavor: default sha1: 70f8415b300f041766fa27faf7d5472699e32388 extra_system_packages: deb: - python3-jmespath - python3-xmltodict - s3cmd rpm: - bzip2 - perl-Test-Harness - python3-jmespath - python3-xmltodict - s3cmd rgw: compression type: random frontend: beast rgw-multisite: realm: is_default: true name: test-realm zonegroups: - enabled_features: - resharding - notification_v2 endpoints: - c1.client.0 is_default: true is_master: true name: a zones: - endpoints: - c1.client.0 is_default: true is_master: true name: a1 - endpoints: - c1.client.1 name: a2 - enabled_features: - resharding - notification_v2 endpoints: - c2.client.0 is_default: true name: b zones: - endpoints: - c2.client.0 is_default: true is_master: true name: b1 - endpoints: - c2.client.1 name: b2 rgw-multisite-tests: args: - tests.py - -a - '!fails_with_rgw' workunit: branch: tt-tentacle sha1: 7b4fb1902b22ff4ea3f4ff6a953bc42198ebeffe owner: kyr priority: 1000 repo: https://github.com/ceph/ceph.git roles: - - c1.mon.a - c1.osd.0 - c1.osd.1 - c1.osd.2 - c1.client.0 - - c1.mgr.x - c1.osd.3 - c1.osd.4 - c1.osd.5 - c1.client.1 - - c2.mon.a - c2.osd.0 - c2.osd.1 - c2.osd.2 - c2.client.0 - - c2.mgr.x - c2.osd.3 - c2.osd.4 - c2.osd.5 - c2.client.1 seed: 7702 sha1: 70f8415b300f041766fa27faf7d5472699e32388 sleep_before_teardown: 0 suite: rgw suite_branch: tt-tentacle suite_path: /home/teuthos/src/github.com_kshtsk_ceph_7b4fb1902b22ff4ea3f4ff6a953bc42198ebeffe/qa suite_relpath: qa suite_repo: https://github.com/kshtsk/ceph.git suite_sha1: 7b4fb1902b22ff4ea3f4ff6a953bc42198ebeffe targets: vm02.local: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHDOdGzmlLQ/ZTYxKoEn2ygRbjgX4ux/lNWXbj7ocH2unOqJgDf8+OP1BWuHYNEv7h7psNGXZ0ZkhElY2KRateg= vm05.local: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIs+bAjIBCO2Z1GDSS25TON+TMyqw5NQcnquTRycytbaJNkFJi5yDkOkkdeCgrmVAOljPOfw/yGvTfVtaKsCVhw= vm07.local: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDlMCt1zzzTlX2IlgevpvFzoeoxOiYD4Qk8mFXF1w3snFJ7povvBC+RimEI07rEIHxU3hlLnvg+Tit+Yf5M5DAs= vm08.local: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNug/VtQYdbJ4Dz5jL/XJ5LvMMhWwcvBq6Amz05ZwPG64TWDh0niEw8yvtwAlDN8ZBITlJN1keEDVaud9T22nD4= tasks: - install: null - ceph: cluster: c1 - ceph: cluster: c2 - rgw: c1.client.0: port: 8000 c1.client.1: port: 8001 c2.client.0: port: 8000 c2.client.1: port: 8001 - rgw-multisite: null - rgw-multisite-tests: config: reconfigure_delay: 90 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: clyso-debian-13 teuthology_repo: https://github.com/clyso/teuthology teuthology_sha1: 1c580df7a9c7c2aadc272da296344fd99f27c444 timestamp: 2026-03-20_10:58:43 tube: vps user: kyr verbose: false worker_log: /home/teuthos/.teuthology/dispatcher/dispatcher.vps.4188345 2026-03-20T11:17:30.306 INFO:teuthology.run:suite_path is set to /home/teuthos/src/github.com_kshtsk_ceph_7b4fb1902b22ff4ea3f4ff6a953bc42198ebeffe/qa; will attempt to use it 2026-03-20T11:17:30.307 INFO:teuthology.run:Found tasks at /home/teuthos/src/github.com_kshtsk_ceph_7b4fb1902b22ff4ea3f4ff6a953bc42198ebeffe/qa/tasks 2026-03-20T11:17:30.307 INFO:teuthology.run_tasks:Running task internal.check_packages... 2026-03-20T11:17:30.307 INFO:teuthology.task.internal:Checking packages... 2026-03-20T11:17:30.307 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '70f8415b300f041766fa27faf7d5472699e32388' 2026-03-20T11:17:30.307 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2026-03-20T11:17:30.307 INFO:teuthology.packaging:ref: None 2026-03-20T11:17:30.307 INFO:teuthology.packaging:tag: None 2026-03-20T11:17:30.308 INFO:teuthology.packaging:branch: tentacle 2026-03-20T11:17:30.308 INFO:teuthology.packaging:sha1: 70f8415b300f041766fa27faf7d5472699e32388 2026-03-20T11:17:30.308 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=tentacle 2026-03-20T11:17:31.041 INFO:teuthology.task.internal:Found packages for ceph version 20.2.0-721.g5bb32787 2026-03-20T11:17:31.043 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2026-03-20T11:17:31.043 INFO:teuthology.task.internal:no buildpackages task found 2026-03-20T11:17:31.044 INFO:teuthology.run_tasks:Running task internal.save_config... 2026-03-20T11:17:31.044 INFO:teuthology.task.internal:Saving configuration 2026-03-20T11:17:31.051 INFO:teuthology.run_tasks:Running task internal.check_lock... 2026-03-20T11:17:31.052 INFO:teuthology.task.internal.check_lock:Checking locks... 2026-03-20T11:17:31.058 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'vm02.local', 'description': '/archive/kyr-2026-03-20_10:58:43-rgw-tentacle-none-default-vps/2068', 'up': True, 'machine_type': 'vps', 'is_vm': True, 'vm_host': {'name': 'localhost', 'description': None, 'up': True, 'machine_type': 'libvirt', 'is_vm': False, 'vm_host': None, 'os_type': None, 'os_version': None, 'arch': None, 'locked': True, 'locked_since': None, 'locked_by': None, 'mac_address': None, 'ssh_pub_key': None}, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-03-20 11:15:06.626977', 'locked_by': 'kyr', 'mac_address': '52:55:00:00:00:02', 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHDOdGzmlLQ/ZTYxKoEn2ygRbjgX4ux/lNWXbj7ocH2unOqJgDf8+OP1BWuHYNEv7h7psNGXZ0ZkhElY2KRateg='} 2026-03-20T11:17:31.063 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'vm05.local', 'description': '/archive/kyr-2026-03-20_10:58:43-rgw-tentacle-none-default-vps/2068', 'up': True, 'machine_type': 'vps', 'is_vm': True, 'vm_host': {'name': 'localhost', 'description': None, 'up': True, 'machine_type': 'libvirt', 'is_vm': False, 'vm_host': None, 'os_type': None, 'os_version': None, 'arch': None, 'locked': True, 'locked_since': None, 'locked_by': None, 'mac_address': None, 'ssh_pub_key': None}, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-03-20 11:15:06.627911', 'locked_by': 'kyr', 'mac_address': '52:55:00:00:00:05', 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIs+bAjIBCO2Z1GDSS25TON+TMyqw5NQcnquTRycytbaJNkFJi5yDkOkkdeCgrmVAOljPOfw/yGvTfVtaKsCVhw='} 2026-03-20T11:17:31.068 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'vm07.local', 'description': '/archive/kyr-2026-03-20_10:58:43-rgw-tentacle-none-default-vps/2068', 'up': True, 'machine_type': 'vps', 'is_vm': True, 'vm_host': {'name': 'localhost', 'description': None, 'up': True, 'machine_type': 'libvirt', 'is_vm': False, 'vm_host': None, 'os_type': None, 'os_version': None, 'arch': None, 'locked': True, 'locked_since': None, 'locked_by': None, 'mac_address': None, 'ssh_pub_key': None}, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-03-20 11:15:06.627699', 'locked_by': 'kyr', 'mac_address': '52:55:00:00:00:07', 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDlMCt1zzzTlX2IlgevpvFzoeoxOiYD4Qk8mFXF1w3snFJ7povvBC+RimEI07rEIHxU3hlLnvg+Tit+Yf5M5DAs='} 2026-03-20T11:17:31.075 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'vm08.local', 'description': '/archive/kyr-2026-03-20_10:58:43-rgw-tentacle-none-default-vps/2068', 'up': True, 'machine_type': 'vps', 'is_vm': True, 'vm_host': {'name': 'localhost', 'description': None, 'up': True, 'machine_type': 'libvirt', 'is_vm': False, 'vm_host': None, 'os_type': None, 'os_version': None, 'arch': None, 'locked': True, 'locked_since': None, 'locked_by': None, 'mac_address': None, 'ssh_pub_key': None}, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-03-20 11:15:06.627451', 'locked_by': 'kyr', 'mac_address': '52:55:00:00:00:08', 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNug/VtQYdbJ4Dz5jL/XJ5LvMMhWwcvBq6Amz05ZwPG64TWDh0niEw8yvtwAlDN8ZBITlJN1keEDVaud9T22nD4='} 2026-03-20T11:17:31.076 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2026-03-20T11:17:31.077 INFO:teuthology.task.internal:roles: ubuntu@vm02.local - ['c1.mon.a', 'c1.osd.0', 'c1.osd.1', 'c1.osd.2', 'c1.client.0'] 2026-03-20T11:17:31.077 INFO:teuthology.task.internal:roles: ubuntu@vm05.local - ['c1.mgr.x', 'c1.osd.3', 'c1.osd.4', 'c1.osd.5', 'c1.client.1'] 2026-03-20T11:17:31.077 INFO:teuthology.task.internal:roles: ubuntu@vm07.local - ['c2.mon.a', 'c2.osd.0', 'c2.osd.1', 'c2.osd.2', 'c2.client.0'] 2026-03-20T11:17:31.077 INFO:teuthology.task.internal:roles: ubuntu@vm08.local - ['c2.mgr.x', 'c2.osd.3', 'c2.osd.4', 'c2.osd.5', 'c2.client.1'] 2026-03-20T11:17:31.077 INFO:teuthology.run_tasks:Running task console_log... 2026-03-20T11:17:31.085 DEBUG:teuthology.task.console_log:vm02 does not support IPMI; excluding 2026-03-20T11:17:31.092 DEBUG:teuthology.task.console_log:vm05 does not support IPMI; excluding 2026-03-20T11:17:31.101 DEBUG:teuthology.task.console_log:vm07 does not support IPMI; excluding 2026-03-20T11:17:31.109 DEBUG:teuthology.task.console_log:vm08 does not support IPMI; excluding 2026-03-20T11:17:31.110 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fb1996f4700>, signals=[15]) 2026-03-20T11:17:31.110 INFO:teuthology.run_tasks:Running task internal.connect... 2026-03-20T11:17:31.111 INFO:teuthology.task.internal:Opening connections... 2026-03-20T11:17:31.111 DEBUG:teuthology.task.internal:connecting to ubuntu@vm02.local 2026-03-20T11:17:31.112 DEBUG:teuthology.orchestra.connection:{'hostname': 'vm02.local', 'username': 'ubuntu', 'timeout': 60} 2026-03-20T11:17:31.169 DEBUG:teuthology.task.internal:connecting to ubuntu@vm05.local 2026-03-20T11:17:31.170 DEBUG:teuthology.orchestra.connection:{'hostname': 'vm05.local', 'username': 'ubuntu', 'timeout': 60} 2026-03-20T11:17:31.228 DEBUG:teuthology.task.internal:connecting to ubuntu@vm07.local 2026-03-20T11:17:31.228 DEBUG:teuthology.orchestra.connection:{'hostname': 'vm07.local', 'username': 'ubuntu', 'timeout': 60} 2026-03-20T11:17:31.289 DEBUG:teuthology.task.internal:connecting to ubuntu@vm08.local 2026-03-20T11:17:31.290 DEBUG:teuthology.orchestra.connection:{'hostname': 'vm08.local', 'username': 'ubuntu', 'timeout': 60} 2026-03-20T11:17:31.351 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2026-03-20T11:17:31.353 DEBUG:teuthology.orchestra.run.vm02:> uname -m 2026-03-20T11:17:31.371 INFO:teuthology.orchestra.run.vm02.stdout:x86_64 2026-03-20T11:17:31.371 DEBUG:teuthology.orchestra.run.vm02:> cat /etc/os-release 2026-03-20T11:17:31.428 INFO:teuthology.orchestra.run.vm02.stdout:NAME="CentOS Stream" 2026-03-20T11:17:31.428 INFO:teuthology.orchestra.run.vm02.stdout:VERSION="9" 2026-03-20T11:17:31.428 INFO:teuthology.orchestra.run.vm02.stdout:ID="centos" 2026-03-20T11:17:31.428 INFO:teuthology.orchestra.run.vm02.stdout:ID_LIKE="rhel fedora" 2026-03-20T11:17:31.428 INFO:teuthology.orchestra.run.vm02.stdout:VERSION_ID="9" 2026-03-20T11:17:31.428 INFO:teuthology.orchestra.run.vm02.stdout:PLATFORM_ID="platform:el9" 2026-03-20T11:17:31.428 INFO:teuthology.orchestra.run.vm02.stdout:PRETTY_NAME="CentOS Stream 9" 2026-03-20T11:17:31.428 INFO:teuthology.orchestra.run.vm02.stdout:ANSI_COLOR="0;31" 2026-03-20T11:17:31.428 INFO:teuthology.orchestra.run.vm02.stdout:LOGO="fedora-logo-icon" 2026-03-20T11:17:31.428 INFO:teuthology.orchestra.run.vm02.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2026-03-20T11:17:31.428 INFO:teuthology.orchestra.run.vm02.stdout:HOME_URL="https://centos.org/" 2026-03-20T11:17:31.428 INFO:teuthology.orchestra.run.vm02.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2026-03-20T11:17:31.428 INFO:teuthology.orchestra.run.vm02.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2026-03-20T11:17:31.428 INFO:teuthology.orchestra.run.vm02.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2026-03-20T11:17:31.428 INFO:teuthology.lock.ops:Updating vm02.local on lock server 2026-03-20T11:17:31.434 DEBUG:teuthology.orchestra.run.vm05:> uname -m 2026-03-20T11:17:31.456 INFO:teuthology.orchestra.run.vm05.stdout:x86_64 2026-03-20T11:17:31.456 DEBUG:teuthology.orchestra.run.vm05:> cat /etc/os-release 2026-03-20T11:17:31.513 INFO:teuthology.orchestra.run.vm05.stdout:NAME="CentOS Stream" 2026-03-20T11:17:31.513 INFO:teuthology.orchestra.run.vm05.stdout:VERSION="9" 2026-03-20T11:17:31.513 INFO:teuthology.orchestra.run.vm05.stdout:ID="centos" 2026-03-20T11:17:31.513 INFO:teuthology.orchestra.run.vm05.stdout:ID_LIKE="rhel fedora" 2026-03-20T11:17:31.513 INFO:teuthology.orchestra.run.vm05.stdout:VERSION_ID="9" 2026-03-20T11:17:31.513 INFO:teuthology.orchestra.run.vm05.stdout:PLATFORM_ID="platform:el9" 2026-03-20T11:17:31.513 INFO:teuthology.orchestra.run.vm05.stdout:PRETTY_NAME="CentOS Stream 9" 2026-03-20T11:17:31.513 INFO:teuthology.orchestra.run.vm05.stdout:ANSI_COLOR="0;31" 2026-03-20T11:17:31.513 INFO:teuthology.orchestra.run.vm05.stdout:LOGO="fedora-logo-icon" 2026-03-20T11:17:31.513 INFO:teuthology.orchestra.run.vm05.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2026-03-20T11:17:31.513 INFO:teuthology.orchestra.run.vm05.stdout:HOME_URL="https://centos.org/" 2026-03-20T11:17:31.513 INFO:teuthology.orchestra.run.vm05.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2026-03-20T11:17:31.513 INFO:teuthology.orchestra.run.vm05.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2026-03-20T11:17:31.513 INFO:teuthology.orchestra.run.vm05.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2026-03-20T11:17:31.514 INFO:teuthology.lock.ops:Updating vm05.local on lock server 2026-03-20T11:17:31.518 DEBUG:teuthology.orchestra.run.vm07:> uname -m 2026-03-20T11:17:31.540 INFO:teuthology.orchestra.run.vm07.stdout:x86_64 2026-03-20T11:17:31.540 DEBUG:teuthology.orchestra.run.vm07:> cat /etc/os-release 2026-03-20T11:17:31.600 INFO:teuthology.orchestra.run.vm07.stdout:NAME="CentOS Stream" 2026-03-20T11:17:31.600 INFO:teuthology.orchestra.run.vm07.stdout:VERSION="9" 2026-03-20T11:17:31.600 INFO:teuthology.orchestra.run.vm07.stdout:ID="centos" 2026-03-20T11:17:31.600 INFO:teuthology.orchestra.run.vm07.stdout:ID_LIKE="rhel fedora" 2026-03-20T11:17:31.600 INFO:teuthology.orchestra.run.vm07.stdout:VERSION_ID="9" 2026-03-20T11:17:31.600 INFO:teuthology.orchestra.run.vm07.stdout:PLATFORM_ID="platform:el9" 2026-03-20T11:17:31.600 INFO:teuthology.orchestra.run.vm07.stdout:PRETTY_NAME="CentOS Stream 9" 2026-03-20T11:17:31.600 INFO:teuthology.orchestra.run.vm07.stdout:ANSI_COLOR="0;31" 2026-03-20T11:17:31.600 INFO:teuthology.orchestra.run.vm07.stdout:LOGO="fedora-logo-icon" 2026-03-20T11:17:31.600 INFO:teuthology.orchestra.run.vm07.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2026-03-20T11:17:31.600 INFO:teuthology.orchestra.run.vm07.stdout:HOME_URL="https://centos.org/" 2026-03-20T11:17:31.600 INFO:teuthology.orchestra.run.vm07.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2026-03-20T11:17:31.601 INFO:teuthology.orchestra.run.vm07.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2026-03-20T11:17:31.601 INFO:teuthology.orchestra.run.vm07.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2026-03-20T11:17:31.601 INFO:teuthology.lock.ops:Updating vm07.local on lock server 2026-03-20T11:17:31.606 DEBUG:teuthology.orchestra.run.vm08:> uname -m 2026-03-20T11:17:31.626 INFO:teuthology.orchestra.run.vm08.stdout:x86_64 2026-03-20T11:17:31.626 DEBUG:teuthology.orchestra.run.vm08:> cat /etc/os-release 2026-03-20T11:17:31.687 INFO:teuthology.orchestra.run.vm08.stdout:NAME="CentOS Stream" 2026-03-20T11:17:31.687 INFO:teuthology.orchestra.run.vm08.stdout:VERSION="9" 2026-03-20T11:17:31.687 INFO:teuthology.orchestra.run.vm08.stdout:ID="centos" 2026-03-20T11:17:31.687 INFO:teuthology.orchestra.run.vm08.stdout:ID_LIKE="rhel fedora" 2026-03-20T11:17:31.687 INFO:teuthology.orchestra.run.vm08.stdout:VERSION_ID="9" 2026-03-20T11:17:31.687 INFO:teuthology.orchestra.run.vm08.stdout:PLATFORM_ID="platform:el9" 2026-03-20T11:17:31.687 INFO:teuthology.orchestra.run.vm08.stdout:PRETTY_NAME="CentOS Stream 9" 2026-03-20T11:17:31.687 INFO:teuthology.orchestra.run.vm08.stdout:ANSI_COLOR="0;31" 2026-03-20T11:17:31.687 INFO:teuthology.orchestra.run.vm08.stdout:LOGO="fedora-logo-icon" 2026-03-20T11:17:31.687 INFO:teuthology.orchestra.run.vm08.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2026-03-20T11:17:31.687 INFO:teuthology.orchestra.run.vm08.stdout:HOME_URL="https://centos.org/" 2026-03-20T11:17:31.687 INFO:teuthology.orchestra.run.vm08.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2026-03-20T11:17:31.687 INFO:teuthology.orchestra.run.vm08.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2026-03-20T11:17:31.687 INFO:teuthology.orchestra.run.vm08.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2026-03-20T11:17:31.688 INFO:teuthology.lock.ops:Updating vm08.local on lock server 2026-03-20T11:17:31.692 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2026-03-20T11:17:31.695 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2026-03-20T11:17:31.697 INFO:teuthology.task.internal:Checking for old test directory... 2026-03-20T11:17:31.697 DEBUG:teuthology.orchestra.run.vm02:> test '!' -e /home/ubuntu/cephtest 2026-03-20T11:17:31.699 DEBUG:teuthology.orchestra.run.vm05:> test '!' -e /home/ubuntu/cephtest 2026-03-20T11:17:31.701 DEBUG:teuthology.orchestra.run.vm07:> test '!' -e /home/ubuntu/cephtest 2026-03-20T11:17:31.703 DEBUG:teuthology.orchestra.run.vm08:> test '!' -e /home/ubuntu/cephtest 2026-03-20T11:17:31.743 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2026-03-20T11:17:31.745 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2026-03-20T11:17:31.745 DEBUG:teuthology.orchestra.run.vm02:> test -z $(ls -A /var/lib/ceph) 2026-03-20T11:17:31.755 DEBUG:teuthology.orchestra.run.vm05:> test -z $(ls -A /var/lib/ceph) 2026-03-20T11:17:31.758 DEBUG:teuthology.orchestra.run.vm07:> test -z $(ls -A /var/lib/ceph) 2026-03-20T11:17:31.763 DEBUG:teuthology.orchestra.run.vm08:> test -z $(ls -A /var/lib/ceph) 2026-03-20T11:17:31.769 INFO:teuthology.orchestra.run.vm02.stderr:ls: cannot access '/var/lib/ceph': No such file or directory 2026-03-20T11:17:31.773 INFO:teuthology.orchestra.run.vm05.stderr:ls: cannot access '/var/lib/ceph': No such file or directory 2026-03-20T11:17:31.780 INFO:teuthology.orchestra.run.vm07.stderr:ls: cannot access '/var/lib/ceph': No such file or directory 2026-03-20T11:17:31.799 INFO:teuthology.orchestra.run.vm08.stderr:ls: cannot access '/var/lib/ceph': No such file or directory 2026-03-20T11:17:31.800 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2026-03-20T11:17:31.809 DEBUG:teuthology.orchestra.run.vm02:> test -e /ceph-qa-ready 2026-03-20T11:17:31.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-20T11:17:32.028 DEBUG:teuthology.orchestra.run.vm05:> test -e /ceph-qa-ready 2026-03-20T11:17:32.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-20T11:17:32.265 DEBUG:teuthology.orchestra.run.vm07:> test -e /ceph-qa-ready 2026-03-20T11:17:32.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-20T11:17:32.499 DEBUG:teuthology.orchestra.run.vm08:> test -e /ceph-qa-ready 2026-03-20T11:17:32.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-20T11:17:32.722 INFO:teuthology.run_tasks:Running task internal.base... 2026-03-20T11:17:32.723 INFO:teuthology.task.internal:Creating test directory... 2026-03-20T11:17:32.723 DEBUG:teuthology.orchestra.run.vm02:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-03-20T11:17:32.726 DEBUG:teuthology.orchestra.run.vm05:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-03-20T11:17:32.728 DEBUG:teuthology.orchestra.run.vm07:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-03-20T11:17:32.730 DEBUG:teuthology.orchestra.run.vm08:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-03-20T11:17:32.749 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2026-03-20T11:17:32.751 INFO:teuthology.run_tasks:Running task internal.archive... 2026-03-20T11:17:32.752 INFO:teuthology.task.internal:Creating archive directory... 2026-03-20T11:17:32.753 DEBUG:teuthology.orchestra.run.vm02:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-03-20T11:17:32.785 DEBUG:teuthology.orchestra.run.vm05:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-03-20T11:17:32.787 DEBUG:teuthology.orchestra.run.vm07:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-03-20T11:17:32.789 DEBUG:teuthology.orchestra.run.vm08:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-03-20T11:17:32.810 INFO:teuthology.run_tasks:Running task internal.coredump... 2026-03-20T11:17:32.812 INFO:teuthology.task.internal:Enabling coredump saving... 2026-03-20T11:17:32.812 DEBUG:teuthology.orchestra.run.vm02:> test -f /run/.containerenv -o -f /.dockerenv 2026-03-20T11:17:32.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-20T11:17:32.860 DEBUG:teuthology.orchestra.run.vm05:> test -f /run/.containerenv -o -f /.dockerenv 2026-03-20T11:17:32.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-20T11:17:32.877 DEBUG:teuthology.orchestra.run.vm07:> test -f /run/.containerenv -o -f /.dockerenv 2026-03-20T11:17:32.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-20T11:17:32.896 DEBUG:teuthology.orchestra.run.vm08:> test -f /run/.containerenv -o -f /.dockerenv 2026-03-20T11:17:32.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-20T11:17:32.910 DEBUG:teuthology.orchestra.run.vm02:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-03-20T11:17:32.912 DEBUG:teuthology.orchestra.run.vm05:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-03-20T11:17:32.919 DEBUG:teuthology.orchestra.run.vm07:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-03-20T11:17:32.940 INFO:teuthology.orchestra.run.vm02.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-03-20T11:17:32.940 DEBUG:teuthology.orchestra.run.vm08:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-03-20T11:17:32.949 INFO:teuthology.orchestra.run.vm05.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-03-20T11:17:32.952 INFO:teuthology.orchestra.run.vm02.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-03-20T11:17:32.964 INFO:teuthology.orchestra.run.vm05.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-03-20T11:17:32.971 INFO:teuthology.orchestra.run.vm07.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-03-20T11:17:32.978 INFO:teuthology.orchestra.run.vm08.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-03-20T11:17:32.984 INFO:teuthology.orchestra.run.vm07.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-03-20T11:17:32.987 INFO:teuthology.orchestra.run.vm08.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-03-20T11:17:32.990 INFO:teuthology.run_tasks:Running task internal.sudo... 2026-03-20T11:17:33.002 INFO:teuthology.task.internal:Configuring sudo... 2026-03-20T11:17:33.002 DEBUG:teuthology.orchestra.run.vm02:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-03-20T11:17:33.004 DEBUG:teuthology.orchestra.run.vm05:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-03-20T11:17:33.009 DEBUG:teuthology.orchestra.run.vm07:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-03-20T11:17:33.030 DEBUG:teuthology.orchestra.run.vm08:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-03-20T11:17:33.059 INFO:teuthology.run_tasks:Running task internal.syslog... 2026-03-20T11:17:33.061 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2026-03-20T11:17:33.061 DEBUG:teuthology.orchestra.run.vm02:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-03-20T11:17:33.074 DEBUG:teuthology.orchestra.run.vm05:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-03-20T11:17:33.084 DEBUG:teuthology.orchestra.run.vm07:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-03-20T11:17:33.102 DEBUG:teuthology.orchestra.run.vm08:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-03-20T11:17:33.118 DEBUG:teuthology.orchestra.run.vm02:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-03-20T11:17:33.159 DEBUG:teuthology.orchestra.run.vm02:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-03-20T11:17:33.217 DEBUG:teuthology.orchestra.run.vm02:> set -ex 2026-03-20T11:17:33.217 DEBUG:teuthology.orchestra.run.vm02:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-03-20T11:17:33.277 DEBUG:teuthology.orchestra.run.vm05:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-03-20T11:17:33.308 DEBUG:teuthology.orchestra.run.vm05:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-03-20T11:17:33.370 DEBUG:teuthology.orchestra.run.vm05:> set -ex 2026-03-20T11:17:33.370 DEBUG:teuthology.orchestra.run.vm05:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-03-20T11:17:33.435 DEBUG:teuthology.orchestra.run.vm07:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-03-20T11:17:33.462 DEBUG:teuthology.orchestra.run.vm07:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-03-20T11:17:33.522 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-20T11:17:33.522 DEBUG:teuthology.orchestra.run.vm07:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-03-20T11:17:33.593 DEBUG:teuthology.orchestra.run.vm08:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-03-20T11:17:33.618 DEBUG:teuthology.orchestra.run.vm08:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-03-20T11:17:33.677 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-20T11:17:33.677 DEBUG:teuthology.orchestra.run.vm08:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-03-20T11:17:33.740 DEBUG:teuthology.orchestra.run.vm02:> sudo service rsyslog restart 2026-03-20T11:17:33.742 DEBUG:teuthology.orchestra.run.vm05:> sudo service rsyslog restart 2026-03-20T11:17:33.745 DEBUG:teuthology.orchestra.run.vm07:> sudo service rsyslog restart 2026-03-20T11:17:33.747 DEBUG:teuthology.orchestra.run.vm08:> sudo service rsyslog restart 2026-03-20T11:17:33.768 INFO:teuthology.orchestra.run.vm02.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-03-20T11:17:33.778 INFO:teuthology.orchestra.run.vm05.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-03-20T11:17:33.782 INFO:teuthology.orchestra.run.vm07.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-03-20T11:17:33.814 INFO:teuthology.orchestra.run.vm08.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-03-20T11:17:34.228 INFO:teuthology.run_tasks:Running task internal.timer... 2026-03-20T11:17:34.230 INFO:teuthology.task.internal:Starting timer... 2026-03-20T11:17:34.230 INFO:teuthology.run_tasks:Running task pcp... 2026-03-20T11:17:34.233 INFO:teuthology.run_tasks:Running task selinux... 2026-03-20T11:17:34.236 INFO:teuthology.task.selinux:Excluding vm02: VMs are not yet supported 2026-03-20T11:17:34.236 INFO:teuthology.task.selinux:Excluding vm05: VMs are not yet supported 2026-03-20T11:17:34.236 INFO:teuthology.task.selinux:Excluding vm07: VMs are not yet supported 2026-03-20T11:17:34.236 INFO:teuthology.task.selinux:Excluding vm08: VMs are not yet supported 2026-03-20T11:17:34.236 DEBUG:teuthology.task.selinux:Getting current SELinux state 2026-03-20T11:17:34.236 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2026-03-20T11:17:34.236 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2026-03-20T11:17:34.236 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2026-03-20T11:17:34.237 DEBUG:teuthology.task:Applying overrides for task ansible.cephlab: {'branch': 'main', 'repo': 'https://github.com/kshtsk/ceph-cm-ansible.git', 'skip_tags': 'nagios,monitoring-scripts,hostname,pubkeys,zap,sudoers,kerberos,ntp-client,resolvconf,cpan,nfs', 'vars': {'logical_volumes': {'lv_1': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}, 'lv_2': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}, 'lv_3': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}, 'lv_4': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}}, 'timezone': 'UTC', 'volume_groups': {'vg_nvme': {'pvs': '/dev/vdb,/dev/vdc,/dev/vdd,/dev/vde'}}}} 2026-03-20T11:17:34.238 DEBUG:teuthology.repo_utils:Setting repo remote to https://github.com/kshtsk/ceph-cm-ansible.git 2026-03-20T11:17:34.240 INFO:teuthology.repo_utils:Fetching github.com_kshtsk_ceph-cm-ansible_main from origin 2026-03-20T11:17:34.935 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthos/src/github.com_kshtsk_ceph-cm-ansible_main to origin/main 2026-03-20T11:17:34.944 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2026-03-20T11:17:34.945 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu", "logical_volumes": {"lv_1": {"scratch_dev": true, "size": "25%VG", "vg": "vg_nvme"}, "lv_2": {"scratch_dev": true, "size": "25%VG", "vg": "vg_nvme"}, "lv_3": {"scratch_dev": true, "size": "25%VG", "vg": "vg_nvme"}, "lv_4": {"scratch_dev": true, "size": "25%VG", "vg": "vg_nvme"}}, "timezone": "UTC", "volume_groups": {"vg_nvme": {"pvs": "/dev/vdb,/dev/vdc,/dev/vdd,/dev/vde"}}}' -i /tmp/teuth_ansible_inventoryioqv7lw0 --limit vm02.local,vm05.local,vm07.local,vm08.local /home/teuthos/src/github.com_kshtsk_ceph-cm-ansible_main/cephlab.yml --skip-tags nagios,monitoring-scripts,hostname,pubkeys,zap,sudoers,kerberos,ntp-client,resolvconf,cpan,nfs 2026-03-20T11:20:04.514 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@vm02.local'), Remote(name='ubuntu@vm05.local'), Remote(name='ubuntu@vm07.local'), Remote(name='ubuntu@vm08.local')] 2026-03-20T11:20:04.514 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@vm02.local' 2026-03-20T11:20:04.515 DEBUG:teuthology.orchestra.connection:{'hostname': 'vm02.local', 'username': 'ubuntu', 'timeout': 60} 2026-03-20T11:20:04.572 DEBUG:teuthology.orchestra.run.vm02:> true 2026-03-20T11:20:04.656 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@vm02.local' 2026-03-20T11:20:04.656 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@vm05.local' 2026-03-20T11:20:04.656 DEBUG:teuthology.orchestra.connection:{'hostname': 'vm05.local', 'username': 'ubuntu', 'timeout': 60} 2026-03-20T11:20:04.725 DEBUG:teuthology.orchestra.run.vm05:> true 2026-03-20T11:20:04.804 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@vm05.local' 2026-03-20T11:20:04.804 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@vm07.local' 2026-03-20T11:20:04.804 DEBUG:teuthology.orchestra.connection:{'hostname': 'vm07.local', 'username': 'ubuntu', 'timeout': 60} 2026-03-20T11:20:04.870 DEBUG:teuthology.orchestra.run.vm07:> true 2026-03-20T11:20:04.962 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@vm07.local' 2026-03-20T11:20:04.963 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@vm08.local' 2026-03-20T11:20:04.963 DEBUG:teuthology.orchestra.connection:{'hostname': 'vm08.local', 'username': 'ubuntu', 'timeout': 60} 2026-03-20T11:20:05.026 DEBUG:teuthology.orchestra.run.vm08:> true 2026-03-20T11:20:05.116 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@vm08.local' 2026-03-20T11:20:05.116 INFO:teuthology.run_tasks:Running task clock... 2026-03-20T11:20:05.118 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2026-03-20T11:20:05.118 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-03-20T11:20:05.119 DEBUG:teuthology.orchestra.run.vm02:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-03-20T11:20:05.120 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-03-20T11:20:05.120 DEBUG:teuthology.orchestra.run.vm05:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-03-20T11:20:05.122 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-03-20T11:20:05.122 DEBUG:teuthology.orchestra.run.vm07:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-03-20T11:20:05.125 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-03-20T11:20:05.125 DEBUG:teuthology.orchestra.run.vm08:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-03-20T11:20:05.154 INFO:teuthology.orchestra.run.vm02.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2026-03-20T11:20:05.160 INFO:teuthology.orchestra.run.vm05.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2026-03-20T11:20:05.165 INFO:teuthology.orchestra.run.vm07.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2026-03-20T11:20:05.170 INFO:teuthology.orchestra.run.vm02.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2026-03-20T11:20:05.179 INFO:teuthology.orchestra.run.vm05.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2026-03-20T11:20:05.185 INFO:teuthology.orchestra.run.vm07.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2026-03-20T11:20:05.194 INFO:teuthology.orchestra.run.vm08.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2026-03-20T11:20:05.201 INFO:teuthology.orchestra.run.vm02.stderr:sudo: ntpd: command not found 2026-03-20T11:20:05.213 INFO:teuthology.orchestra.run.vm08.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2026-03-20T11:20:05.216 INFO:teuthology.orchestra.run.vm05.stderr:sudo: ntpd: command not found 2026-03-20T11:20:05.218 INFO:teuthology.orchestra.run.vm02.stdout:506 Cannot talk to daemon 2026-03-20T11:20:05.218 INFO:teuthology.orchestra.run.vm07.stderr:sudo: ntpd: command not found 2026-03-20T11:20:05.234 INFO:teuthology.orchestra.run.vm07.stdout:506 Cannot talk to daemon 2026-03-20T11:20:05.234 INFO:teuthology.orchestra.run.vm05.stdout:506 Cannot talk to daemon 2026-03-20T11:20:05.236 INFO:teuthology.orchestra.run.vm02.stderr:Failed to start ntp.service: Unit ntp.service not found. 2026-03-20T11:20:05.250 INFO:teuthology.orchestra.run.vm08.stderr:sudo: ntpd: command not found 2026-03-20T11:20:05.252 INFO:teuthology.orchestra.run.vm05.stderr:Failed to start ntp.service: Unit ntp.service not found. 2026-03-20T11:20:05.254 INFO:teuthology.orchestra.run.vm02.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2026-03-20T11:20:05.255 INFO:teuthology.orchestra.run.vm07.stderr:Failed to start ntp.service: Unit ntp.service not found. 2026-03-20T11:20:05.266 INFO:teuthology.orchestra.run.vm08.stdout:506 Cannot talk to daemon 2026-03-20T11:20:05.268 INFO:teuthology.orchestra.run.vm05.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2026-03-20T11:20:05.276 INFO:teuthology.orchestra.run.vm07.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2026-03-20T11:20:05.284 INFO:teuthology.orchestra.run.vm08.stderr:Failed to start ntp.service: Unit ntp.service not found. 2026-03-20T11:20:05.300 INFO:teuthology.orchestra.run.vm08.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2026-03-20T11:20:05.315 INFO:teuthology.orchestra.run.vm02.stderr:bash: line 1: ntpq: command not found 2026-03-20T11:20:05.317 INFO:teuthology.orchestra.run.vm02.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-03-20T11:20:05.317 INFO:teuthology.orchestra.run.vm02.stdout:=============================================================================== 2026-03-20T11:20:05.334 INFO:teuthology.orchestra.run.vm07.stderr:bash: line 1: ntpq: command not found 2026-03-20T11:20:05.339 INFO:teuthology.orchestra.run.vm07.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-03-20T11:20:05.339 INFO:teuthology.orchestra.run.vm07.stdout:=============================================================================== 2026-03-20T11:20:05.343 INFO:teuthology.orchestra.run.vm05.stderr:bash: line 1: ntpq: command not found 2026-03-20T11:20:05.346 INFO:teuthology.orchestra.run.vm05.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-03-20T11:20:05.347 INFO:teuthology.orchestra.run.vm05.stdout:=============================================================================== 2026-03-20T11:20:05.362 INFO:teuthology.orchestra.run.vm08.stderr:bash: line 1: ntpq: command not found 2026-03-20T11:20:05.365 INFO:teuthology.orchestra.run.vm08.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-03-20T11:20:05.365 INFO:teuthology.orchestra.run.vm08.stdout:=============================================================================== 2026-03-20T11:20:05.365 INFO:teuthology.run_tasks:Running task install... 2026-03-20T11:20:05.368 DEBUG:teuthology.task.install:project ceph 2026-03-20T11:20:05.368 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '70f8415b300f041766fa27faf7d5472699e32388'}, 'extra_system_packages': {'deb': ['python3-jmespath', 'python3-xmltodict', 's3cmd'], 'rpm': ['bzip2', 'perl-Test-Harness', 'python3-jmespath', 'python3-xmltodict', 's3cmd']}} 2026-03-20T11:20:05.368 DEBUG:teuthology.task.install:config {'flavor': 'default', 'sha1': '70f8415b300f041766fa27faf7d5472699e32388', 'extra_system_packages': {'deb': ['python3-jmespath', 'python3-xmltodict', 's3cmd'], 'rpm': ['bzip2', 'perl-Test-Harness', 'python3-jmespath', 'python3-xmltodict', 's3cmd']}} 2026-03-20T11:20:05.368 INFO:teuthology.task.install:Using flavor: default 2026-03-20T11:20:05.371 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2026-03-20T11:20:05.371 INFO:teuthology.task.install:extra packages: [] 2026-03-20T11:20:05.371 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': [], 'extra_system_packages': {'deb': ['python3-jmespath', 'python3-xmltodict', 's3cmd'], 'rpm': ['bzip2', 'perl-Test-Harness', 'python3-jmespath', 'python3-xmltodict', 's3cmd']}, 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '70f8415b300f041766fa27faf7d5472699e32388', 'tag': None, 'wait_for_package': False} 2026-03-20T11:20:05.371 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=70f8415b300f041766fa27faf7d5472699e32388 2026-03-20T11:20:05.372 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': [], 'extra_system_packages': {'deb': ['python3-jmespath', 'python3-xmltodict', 's3cmd'], 'rpm': ['bzip2', 'perl-Test-Harness', 'python3-jmespath', 'python3-xmltodict', 's3cmd']}, 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '70f8415b300f041766fa27faf7d5472699e32388', 'tag': None, 'wait_for_package': False} 2026-03-20T11:20:05.372 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=70f8415b300f041766fa27faf7d5472699e32388 2026-03-20T11:20:05.372 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': [], 'extra_system_packages': {'deb': ['python3-jmespath', 'python3-xmltodict', 's3cmd'], 'rpm': ['bzip2', 'perl-Test-Harness', 'python3-jmespath', 'python3-xmltodict', 's3cmd']}, 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '70f8415b300f041766fa27faf7d5472699e32388', 'tag': None, 'wait_for_package': False} 2026-03-20T11:20:05.372 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=70f8415b300f041766fa27faf7d5472699e32388 2026-03-20T11:20:05.373 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': [], 'extra_system_packages': {'deb': ['python3-jmespath', 'python3-xmltodict', 's3cmd'], 'rpm': ['bzip2', 'perl-Test-Harness', 'python3-jmespath', 'python3-xmltodict', 's3cmd']}, 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '70f8415b300f041766fa27faf7d5472699e32388', 'tag': None, 'wait_for_package': False} 2026-03-20T11:20:05.373 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=70f8415b300f041766fa27faf7d5472699e32388 2026-03-20T11:20:05.963 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/centos/9/flavors/default/ 2026-03-20T11:20:05.963 INFO:teuthology.task.install.rpm:Package version is 20.2.0-712.g70f8415b 2026-03-20T11:20:05.988 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/centos/9/flavors/default/ 2026-03-20T11:20:05.988 INFO:teuthology.task.install.rpm:Package version is 20.2.0-712.g70f8415b 2026-03-20T11:20:06.023 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/centos/9/flavors/default/ 2026-03-20T11:20:06.023 INFO:teuthology.task.install.rpm:Package version is 20.2.0-712.g70f8415b 2026-03-20T11:20:06.117 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/centos/9/flavors/default/ 2026-03-20T11:20:06.117 INFO:teuthology.task.install.rpm:Package version is 20.2.0-712.g70f8415b 2026-03-20T11:20:06.475 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2026-03-20T11:20:06.475 DEBUG:teuthology.orchestra.run.vm02:> set -ex 2026-03-20T11:20:06.475 DEBUG:teuthology.orchestra.run.vm02:> sudo dd of=/etc/yum.repos.d/ceph.repo 2026-03-20T11:20:06.510 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, bzip2, perl-Test-Harness, python3-jmespath, python3-xmltodict, s3cmd on remote rpm x86_64 2026-03-20T11:20:06.510 DEBUG:teuthology.orchestra.run.vm02:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/70f8415b300f041766fa27faf7d5472699e32388/;g' /etc/yum.repos.d/ceph.repo ; fi 2026-03-20T11:20:06.545 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2026-03-20T11:20:06.545 DEBUG:teuthology.orchestra.run.vm05:> set -ex 2026-03-20T11:20:06.545 DEBUG:teuthology.orchestra.run.vm05:> sudo dd of=/etc/yum.repos.d/ceph.repo 2026-03-20T11:20:06.579 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, bzip2, perl-Test-Harness, python3-jmespath, python3-xmltodict, s3cmd on remote rpm x86_64 2026-03-20T11:20:06.579 DEBUG:teuthology.orchestra.run.vm05:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/70f8415b300f041766fa27faf7d5472699e32388/;g' /etc/yum.repos.d/ceph.repo ; fi 2026-03-20T11:20:06.585 DEBUG:teuthology.orchestra.run.vm02:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2026-03-20T11:20:06.587 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2026-03-20T11:20:06.588 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-20T11:20:06.588 DEBUG:teuthology.orchestra.run.vm07:> sudo dd of=/etc/yum.repos.d/ceph.repo 2026-03-20T11:20:06.626 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, bzip2, perl-Test-Harness, python3-jmespath, python3-xmltodict, s3cmd on remote rpm x86_64 2026-03-20T11:20:06.626 DEBUG:teuthology.orchestra.run.vm07:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/70f8415b300f041766fa27faf7d5472699e32388/;g' /etc/yum.repos.d/ceph.repo ; fi 2026-03-20T11:20:06.636 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2026-03-20T11:20:06.636 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-20T11:20:06.636 DEBUG:teuthology.orchestra.run.vm08:> sudo dd of=/etc/yum.repos.d/ceph.repo 2026-03-20T11:20:06.660 DEBUG:teuthology.orchestra.run.vm05:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2026-03-20T11:20:06.670 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, bzip2, perl-Test-Harness, python3-jmespath, python3-xmltodict, s3cmd on remote rpm x86_64 2026-03-20T11:20:06.671 DEBUG:teuthology.orchestra.run.vm08:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/70f8415b300f041766fa27faf7d5472699e32388/;g' /etc/yum.repos.d/ceph.repo ; fi 2026-03-20T11:20:06.671 DEBUG:teuthology.orchestra.run.vm02:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2026-03-20T11:20:06.705 INFO:teuthology.orchestra.run.vm02.stdout:check_obsoletes = 1 2026-03-20T11:20:06.706 DEBUG:teuthology.orchestra.run.vm02:> sudo yum clean all 2026-03-20T11:20:06.709 DEBUG:teuthology.orchestra.run.vm07:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2026-03-20T11:20:06.744 DEBUG:teuthology.orchestra.run.vm08:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2026-03-20T11:20:06.755 DEBUG:teuthology.orchestra.run.vm05:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2026-03-20T11:20:06.788 INFO:teuthology.orchestra.run.vm05.stdout:check_obsoletes = 1 2026-03-20T11:20:06.790 DEBUG:teuthology.orchestra.run.vm05:> sudo yum clean all 2026-03-20T11:20:06.808 DEBUG:teuthology.orchestra.run.vm07:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2026-03-20T11:20:06.828 DEBUG:teuthology.orchestra.run.vm08:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2026-03-20T11:20:06.842 INFO:teuthology.orchestra.run.vm07.stdout:check_obsoletes = 1 2026-03-20T11:20:06.844 DEBUG:teuthology.orchestra.run.vm07:> sudo yum clean all 2026-03-20T11:20:06.861 INFO:teuthology.orchestra.run.vm08.stdout:check_obsoletes = 1 2026-03-20T11:20:06.862 DEBUG:teuthology.orchestra.run.vm08:> sudo yum clean all 2026-03-20T11:20:06.906 INFO:teuthology.orchestra.run.vm02.stdout:41 files removed 2026-03-20T11:20:06.930 DEBUG:teuthology.orchestra.run.vm02:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd bzip2 perl-Test-Harness python3-jmespath python3-xmltodict s3cmd 2026-03-20T11:20:07.019 INFO:teuthology.orchestra.run.vm05.stdout:41 files removed 2026-03-20T11:20:07.062 DEBUG:teuthology.orchestra.run.vm05:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd bzip2 perl-Test-Harness python3-jmespath python3-xmltodict s3cmd 2026-03-20T11:20:07.070 INFO:teuthology.orchestra.run.vm07.stdout:41 files removed 2026-03-20T11:20:07.100 INFO:teuthology.orchestra.run.vm08.stdout:41 files removed 2026-03-20T11:20:07.119 DEBUG:teuthology.orchestra.run.vm07:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd bzip2 perl-Test-Harness python3-jmespath python3-xmltodict s3cmd 2026-03-20T11:20:07.158 DEBUG:teuthology.orchestra.run.vm08:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd bzip2 perl-Test-Harness python3-jmespath python3-xmltodict s3cmd 2026-03-20T11:20:08.334 INFO:teuthology.orchestra.run.vm02.stdout:ceph packages for x86_64 74 kB/s | 87 kB 00:01 2026-03-20T11:20:08.465 INFO:teuthology.orchestra.run.vm05.stdout:ceph packages for x86_64 73 kB/s | 87 kB 00:01 2026-03-20T11:20:08.507 INFO:teuthology.orchestra.run.vm08.stdout:ceph packages for x86_64 76 kB/s | 87 kB 00:01 2026-03-20T11:20:08.522 INFO:teuthology.orchestra.run.vm07.stdout:ceph packages for x86_64 74 kB/s | 87 kB 00:01 2026-03-20T11:20:09.371 INFO:teuthology.orchestra.run.vm02.stdout:ceph noarch packages 18 kB/s | 18 kB 00:01 2026-03-20T11:20:09.543 INFO:teuthology.orchestra.run.vm05.stdout:ceph noarch packages 17 kB/s | 18 kB 00:01 2026-03-20T11:20:09.589 INFO:teuthology.orchestra.run.vm07.stdout:ceph noarch packages 17 kB/s | 18 kB 00:01 2026-03-20T11:20:09.593 INFO:teuthology.orchestra.run.vm08.stdout:ceph noarch packages 17 kB/s | 18 kB 00:01 2026-03-20T11:20:10.341 INFO:teuthology.orchestra.run.vm02.stdout:ceph source packages 2.1 kB/s | 1.9 kB 00:00 2026-03-20T11:20:10.524 INFO:teuthology.orchestra.run.vm05.stdout:ceph source packages 2.0 kB/s | 1.9 kB 00:00 2026-03-20T11:20:10.553 INFO:teuthology.orchestra.run.vm07.stdout:ceph source packages 2.1 kB/s | 1.9 kB 00:00 2026-03-20T11:20:10.592 INFO:teuthology.orchestra.run.vm08.stdout:ceph source packages 2.0 kB/s | 1.9 kB 00:00 2026-03-20T11:20:11.183 INFO:teuthology.orchestra.run.vm02.stdout:CentOS Stream 9 - BaseOS 11 MB/s | 8.9 MB 00:00 2026-03-20T11:20:11.416 INFO:teuthology.orchestra.run.vm07.stdout:CentOS Stream 9 - BaseOS 11 MB/s | 8.9 MB 00:00 2026-03-20T11:20:11.915 INFO:teuthology.orchestra.run.vm05.stdout:CentOS Stream 9 - BaseOS 6.5 MB/s | 8.9 MB 00:01 2026-03-20T11:20:12.485 INFO:teuthology.orchestra.run.vm08.stdout:CentOS Stream 9 - BaseOS 4.8 MB/s | 8.9 MB 00:01 2026-03-20T11:20:13.457 INFO:teuthology.orchestra.run.vm02.stdout:CentOS Stream 9 - AppStream 18 MB/s | 27 MB 00:01 2026-03-20T11:20:13.615 INFO:teuthology.orchestra.run.vm07.stdout:CentOS Stream 9 - AppStream 19 MB/s | 27 MB 00:01 2026-03-20T11:20:14.357 INFO:teuthology.orchestra.run.vm05.stdout:CentOS Stream 9 - AppStream 16 MB/s | 27 MB 00:01 2026-03-20T11:20:14.524 INFO:teuthology.orchestra.run.vm08.stdout:CentOS Stream 9 - AppStream 22 MB/s | 27 MB 00:01 2026-03-20T11:20:17.040 INFO:teuthology.orchestra.run.vm02.stdout:CentOS Stream 9 - CRB 16 MB/s | 8.0 MB 00:00 2026-03-20T11:20:17.524 INFO:teuthology.orchestra.run.vm07.stdout:CentOS Stream 9 - CRB 11 MB/s | 8.0 MB 00:00 2026-03-20T11:20:18.539 INFO:teuthology.orchestra.run.vm02.stdout:CentOS Stream 9 - Extras packages 38 kB/s | 20 kB 00:00 2026-03-20T11:20:18.672 INFO:teuthology.orchestra.run.vm07.stdout:CentOS Stream 9 - Extras packages 79 kB/s | 20 kB 00:00 2026-03-20T11:20:18.747 INFO:teuthology.orchestra.run.vm05.stdout:CentOS Stream 9 - CRB 6.0 MB/s | 8.0 MB 00:01 2026-03-20T11:20:19.814 INFO:teuthology.orchestra.run.vm02.stdout:Extra Packages for Enterprise Linux 17 MB/s | 20 MB 00:01 2026-03-20T11:20:19.909 INFO:teuthology.orchestra.run.vm05.stdout:CentOS Stream 9 - Extras packages 81 kB/s | 20 kB 00:00 2026-03-20T11:20:20.481 INFO:teuthology.orchestra.run.vm07.stdout:Extra Packages for Enterprise Linux 12 MB/s | 20 MB 00:01 2026-03-20T11:20:20.932 INFO:teuthology.orchestra.run.vm05.stdout:Extra Packages for Enterprise Linux 22 MB/s | 20 MB 00:00 2026-03-20T11:20:21.274 INFO:teuthology.orchestra.run.vm08.stdout:CentOS Stream 9 - CRB 2.2 MB/s | 8.0 MB 00:03 2026-03-20T11:20:22.767 INFO:teuthology.orchestra.run.vm08.stdout:CentOS Stream 9 - Extras packages 42 kB/s | 20 kB 00:00 2026-03-20T11:20:25.873 INFO:teuthology.orchestra.run.vm08.stdout:Extra Packages for Enterprise Linux 6.8 MB/s | 20 MB 00:03 2026-03-20T11:20:25.873 INFO:teuthology.orchestra.run.vm02.stdout:lab-extras 58 kB/s | 50 kB 00:00 2026-03-20T11:20:26.980 INFO:teuthology.orchestra.run.vm07.stdout:lab-extras 65 kB/s | 50 kB 00:00 2026-03-20T11:20:27.337 INFO:teuthology.orchestra.run.vm05.stdout:lab-extras 63 kB/s | 50 kB 00:00 2026-03-20T11:20:27.360 INFO:teuthology.orchestra.run.vm02.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2026-03-20T11:20:27.361 INFO:teuthology.orchestra.run.vm02.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2026-03-20T11:20:27.402 INFO:teuthology.orchestra.run.vm02.stdout:Dependencies resolved. 2026-03-20T11:20:27.407 INFO:teuthology.orchestra.run.vm02.stdout:====================================================================================== 2026-03-20T11:20:27.407 INFO:teuthology.orchestra.run.vm02.stdout: Package Arch Version Repository Size 2026-03-20T11:20:27.407 INFO:teuthology.orchestra.run.vm02.stdout:====================================================================================== 2026-03-20T11:20:27.407 INFO:teuthology.orchestra.run.vm02.stdout:Installing: 2026-03-20T11:20:27.407 INFO:teuthology.orchestra.run.vm02.stdout: bzip2 x86_64 1.0.8-11.el9 baseos 55 k 2026-03-20T11:20:27.407 INFO:teuthology.orchestra.run.vm02.stdout: ceph x86_64 2:20.2.0-712.g70f8415b.el9 ceph 6.5 k 2026-03-20T11:20:27.407 INFO:teuthology.orchestra.run.vm02.stdout: ceph-base x86_64 2:20.2.0-712.g70f8415b.el9 ceph 5.9 M 2026-03-20T11:20:27.407 INFO:teuthology.orchestra.run.vm02.stdout: ceph-fuse x86_64 2:20.2.0-712.g70f8415b.el9 ceph 939 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: ceph-immutable-object-cache x86_64 2:20.2.0-712.g70f8415b.el9 ceph 154 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: ceph-mgr x86_64 2:20.2.0-712.g70f8415b.el9 ceph 962 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: ceph-mgr-cephadm noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 173 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: ceph-mgr-dashboard noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 11 M 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: ceph-mgr-diskprediction-local noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 7.4 M 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: ceph-mgr-rook noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 50 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: ceph-radosgw x86_64 2:20.2.0-712.g70f8415b.el9 ceph 24 M 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: ceph-test x86_64 2:20.2.0-712.g70f8415b.el9 ceph 84 M 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: ceph-volume noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 298 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: cephadm noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 1.0 M 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: libcephfs-devel x86_64 2:20.2.0-712.g70f8415b.el9 ceph 34 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: libcephfs2 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 866 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: librados-devel x86_64 2:20.2.0-712.g70f8415b.el9 ceph 126 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: perl-Test-Harness noarch 1:3.42-461.el9 appstream 295 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: python3-cephfs x86_64 2:20.2.0-712.g70f8415b.el9 ceph 163 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: python3-jmespath noarch 1.0.1-1.el9 appstream 48 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: python3-rados x86_64 2:20.2.0-712.g70f8415b.el9 ceph 324 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: python3-rbd x86_64 2:20.2.0-712.g70f8415b.el9 ceph 304 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: python3-rgw x86_64 2:20.2.0-712.g70f8415b.el9 ceph 99 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: python3-xmltodict noarch 0.12.0-15.el9 epel 22 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: rbd-fuse x86_64 2:20.2.0-712.g70f8415b.el9 ceph 91 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: rbd-mirror x86_64 2:20.2.0-712.g70f8415b.el9 ceph 2.9 M 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: rbd-nbd x86_64 2:20.2.0-712.g70f8415b.el9 ceph 180 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: s3cmd noarch 2.4.0-1.el9 epel 206 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout:Upgrading: 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: librados2 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 3.5 M 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: librbd1 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 2.8 M 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout:Installing dependencies: 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: abseil-cpp x86_64 20211102.0-4.el9 epel 551 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: boost-program-options x86_64 1.75.0-13.el9 appstream 104 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: ceph-common x86_64 2:20.2.0-712.g70f8415b.el9 ceph 24 M 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: ceph-grafana-dashboards noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 43 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: ceph-mds x86_64 2:20.2.0-712.g70f8415b.el9 ceph 2.3 M 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: ceph-mgr-modules-core noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 290 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: ceph-mon x86_64 2:20.2.0-712.g70f8415b.el9 ceph 5.0 M 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: ceph-osd x86_64 2:20.2.0-712.g70f8415b.el9 ceph 17 M 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: ceph-prometheus-alerts noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 17 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: ceph-selinux x86_64 2:20.2.0-712.g70f8415b.el9 ceph 25 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: cryptsetup x86_64 2.8.1-3.el9 baseos 351 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: flexiblas x86_64 3.0.4-9.el9 appstream 30 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: flexiblas-netlib x86_64 3.0.4-9.el9 appstream 3.0 M 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: flexiblas-openblas-openmp x86_64 3.0.4-9.el9 appstream 15 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: fuse x86_64 2.9.9-17.el9 baseos 80 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: ledmon-libs x86_64 1.1.0-3.el9 baseos 40 k 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: libarrow x86_64 9.0.0-15.el9 epel 4.4 M 2026-03-20T11:20:27.408 INFO:teuthology.orchestra.run.vm02.stdout: libarrow-doc noarch 9.0.0-15.el9 epel 25 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: libcephfs-proxy2 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 24 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: libcephsqlite x86_64 2:20.2.0-712.g70f8415b.el9 ceph 164 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: libconfig x86_64 1.7.2-9.el9 baseos 72 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: libgfortran x86_64 11.5.0-14.el9 baseos 794 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: libnbd x86_64 1.20.3-4.el9 appstream 164 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: liboath x86_64 2.6.12-1.el9 epel 49 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: libquadmath x86_64 11.5.0-14.el9 baseos 184 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: libradosstriper1 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 250 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: librgw2 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 6.4 M 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: libstoragemgmt x86_64 1.10.1-1.el9 appstream 246 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: libxslt x86_64 1.1.34-12.el9 appstream 233 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: mailcap noarch 2.1.49-5.el9 baseos 33 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: openblas x86_64 0.3.29-1.el9 appstream 42 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: openblas-openmp x86_64 0.3.29-1.el9 appstream 5.3 M 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: parquet-libs x86_64 9.0.0-15.el9 epel 838 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: pciutils x86_64 3.7.0-7.el9 baseos 93 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: perl-Benchmark noarch 1.23-483.el9 appstream 26 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: protobuf x86_64 3.14.0-17.el9 appstream 1.0 M 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: protobuf-compiler x86_64 3.14.0-17.el9 crb 862 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: python3-autocommand noarch 2.2.2-8.el9 epel 29 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: python3-backports-tarfile noarch 1.2.0-1.el9 epel 60 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: python3-ceph-argparse x86_64 2:20.2.0-712.g70f8415b.el9 ceph 45 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: python3-ceph-common x86_64 2:20.2.0-712.g70f8415b.el9 ceph 175 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: python3-cheroot noarch 10.0.1-4.el9 epel 173 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: python3-cryptography x86_64 36.0.1-5.el9 baseos 1.2 M 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: python3-devel x86_64 3.9.25-3.el9 appstream 244 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: python3-google-auth noarch 1:2.45.0-1.el9 epel 254 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2026-03-20T11:20:27.409 INFO:teuthology.orchestra.run.vm02.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-jaraco-context noarch 6.0.1-3.el9 epel 20 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-jaraco-text noarch 4.0.0-2.el9 epel 26 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-jinja2 noarch 2.11.3-8.el9 appstream 249 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-kubernetes noarch 1:26.1.0-3.el9 epel 1.0 M 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-libstoragemgmt x86_64 1.10.1-1.el9 appstream 177 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-numpy x86_64 1:1.23.5-2.el9 appstream 6.1 M 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-numpy-f2py x86_64 1:1.23.5-2.el9 appstream 442 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-protobuf noarch 3.14.0-17.el9 appstream 267 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-pyasn1 noarch 0.4.8-7.el9 appstream 157 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-pyasn1-modules noarch 0.4.8-7.el9 appstream 277 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-pyparsing noarch 2.4.7-9.el9 baseos 150 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-requests noarch 2.25.1-10.el9 baseos 126 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-scipy x86_64 1.9.3-2.el9 appstream 19 M 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-typing-extensions noarch 4.15.0-1.el9 epel 86 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-urllib3 noarch 1.26.5-7.el9 baseos 218 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: qatlib x86_64 25.08.0-2.el9 appstream 240 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: qatzip-libs x86_64 1.3.1-1.el9 appstream 66 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: socat x86_64 1.7.4.1-8.el9 appstream 303 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: thrift x86_64 0.15.0-4.el9 epel 1.6 M 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: unzip x86_64 6.0-59.el9 baseos 182 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: zip x86_64 3.0-35.el9 baseos 266 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout:Installing weak dependencies: 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: qatlib-service x86_64 25.08.0-2.el9 appstream 37 k 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout:Transaction Summary 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout:====================================================================================== 2026-03-20T11:20:27.410 INFO:teuthology.orchestra.run.vm02.stdout:Install 136 Packages 2026-03-20T11:20:27.411 INFO:teuthology.orchestra.run.vm02.stdout:Upgrade 2 Packages 2026-03-20T11:20:27.411 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:20:27.411 INFO:teuthology.orchestra.run.vm02.stdout:Total download size: 267 M 2026-03-20T11:20:27.411 INFO:teuthology.orchestra.run.vm02.stdout:Downloading Packages: 2026-03-20T11:20:28.545 INFO:teuthology.orchestra.run.vm07.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2026-03-20T11:20:28.545 INFO:teuthology.orchestra.run.vm07.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2026-03-20T11:20:28.592 INFO:teuthology.orchestra.run.vm07.stdout:Dependencies resolved. 2026-03-20T11:20:28.597 INFO:teuthology.orchestra.run.vm07.stdout:====================================================================================== 2026-03-20T11:20:28.597 INFO:teuthology.orchestra.run.vm07.stdout: Package Arch Version Repository Size 2026-03-20T11:20:28.597 INFO:teuthology.orchestra.run.vm07.stdout:====================================================================================== 2026-03-20T11:20:28.597 INFO:teuthology.orchestra.run.vm07.stdout:Installing: 2026-03-20T11:20:28.597 INFO:teuthology.orchestra.run.vm07.stdout: bzip2 x86_64 1.0.8-11.el9 baseos 55 k 2026-03-20T11:20:28.597 INFO:teuthology.orchestra.run.vm07.stdout: ceph x86_64 2:20.2.0-712.g70f8415b.el9 ceph 6.5 k 2026-03-20T11:20:28.597 INFO:teuthology.orchestra.run.vm07.stdout: ceph-base x86_64 2:20.2.0-712.g70f8415b.el9 ceph 5.9 M 2026-03-20T11:20:28.597 INFO:teuthology.orchestra.run.vm07.stdout: ceph-fuse x86_64 2:20.2.0-712.g70f8415b.el9 ceph 939 k 2026-03-20T11:20:28.597 INFO:teuthology.orchestra.run.vm07.stdout: ceph-immutable-object-cache x86_64 2:20.2.0-712.g70f8415b.el9 ceph 154 k 2026-03-20T11:20:28.597 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr x86_64 2:20.2.0-712.g70f8415b.el9 ceph 962 k 2026-03-20T11:20:28.597 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-cephadm noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 173 k 2026-03-20T11:20:28.597 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-dashboard noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 11 M 2026-03-20T11:20:28.597 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-diskprediction-local noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 7.4 M 2026-03-20T11:20:28.597 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-rook noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 50 k 2026-03-20T11:20:28.597 INFO:teuthology.orchestra.run.vm07.stdout: ceph-radosgw x86_64 2:20.2.0-712.g70f8415b.el9 ceph 24 M 2026-03-20T11:20:28.597 INFO:teuthology.orchestra.run.vm07.stdout: ceph-test x86_64 2:20.2.0-712.g70f8415b.el9 ceph 84 M 2026-03-20T11:20:28.597 INFO:teuthology.orchestra.run.vm07.stdout: ceph-volume noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 298 k 2026-03-20T11:20:28.597 INFO:teuthology.orchestra.run.vm07.stdout: cephadm noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 1.0 M 2026-03-20T11:20:28.597 INFO:teuthology.orchestra.run.vm07.stdout: libcephfs-devel x86_64 2:20.2.0-712.g70f8415b.el9 ceph 34 k 2026-03-20T11:20:28.597 INFO:teuthology.orchestra.run.vm07.stdout: libcephfs2 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 866 k 2026-03-20T11:20:28.597 INFO:teuthology.orchestra.run.vm07.stdout: librados-devel x86_64 2:20.2.0-712.g70f8415b.el9 ceph 126 k 2026-03-20T11:20:28.597 INFO:teuthology.orchestra.run.vm07.stdout: perl-Test-Harness noarch 1:3.42-461.el9 appstream 295 k 2026-03-20T11:20:28.597 INFO:teuthology.orchestra.run.vm07.stdout: python3-cephfs x86_64 2:20.2.0-712.g70f8415b.el9 ceph 163 k 2026-03-20T11:20:28.597 INFO:teuthology.orchestra.run.vm07.stdout: python3-jmespath noarch 1.0.1-1.el9 appstream 48 k 2026-03-20T11:20:28.597 INFO:teuthology.orchestra.run.vm07.stdout: python3-rados x86_64 2:20.2.0-712.g70f8415b.el9 ceph 324 k 2026-03-20T11:20:28.597 INFO:teuthology.orchestra.run.vm07.stdout: python3-rbd x86_64 2:20.2.0-712.g70f8415b.el9 ceph 304 k 2026-03-20T11:20:28.597 INFO:teuthology.orchestra.run.vm07.stdout: python3-rgw x86_64 2:20.2.0-712.g70f8415b.el9 ceph 99 k 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: python3-xmltodict noarch 0.12.0-15.el9 epel 22 k 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: rbd-fuse x86_64 2:20.2.0-712.g70f8415b.el9 ceph 91 k 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: rbd-mirror x86_64 2:20.2.0-712.g70f8415b.el9 ceph 2.9 M 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: rbd-nbd x86_64 2:20.2.0-712.g70f8415b.el9 ceph 180 k 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: s3cmd noarch 2.4.0-1.el9 epel 206 k 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout:Upgrading: 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: librados2 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 3.5 M 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: librbd1 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 2.8 M 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout:Installing dependencies: 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: abseil-cpp x86_64 20211102.0-4.el9 epel 551 k 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: boost-program-options x86_64 1.75.0-13.el9 appstream 104 k 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: ceph-common x86_64 2:20.2.0-712.g70f8415b.el9 ceph 24 M 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: ceph-grafana-dashboards noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 43 k 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mds x86_64 2:20.2.0-712.g70f8415b.el9 ceph 2.3 M 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-modules-core noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 290 k 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mon x86_64 2:20.2.0-712.g70f8415b.el9 ceph 5.0 M 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: ceph-osd x86_64 2:20.2.0-712.g70f8415b.el9 ceph 17 M 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: ceph-prometheus-alerts noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 17 k 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: ceph-selinux x86_64 2:20.2.0-712.g70f8415b.el9 ceph 25 k 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: cryptsetup x86_64 2.8.1-3.el9 baseos 351 k 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: flexiblas x86_64 3.0.4-9.el9 appstream 30 k 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: flexiblas-netlib x86_64 3.0.4-9.el9 appstream 3.0 M 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: flexiblas-openblas-openmp x86_64 3.0.4-9.el9 appstream 15 k 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: fuse x86_64 2.9.9-17.el9 baseos 80 k 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: ledmon-libs x86_64 1.1.0-3.el9 baseos 40 k 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: libarrow x86_64 9.0.0-15.el9 epel 4.4 M 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: libarrow-doc noarch 9.0.0-15.el9 epel 25 k 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: libcephfs-proxy2 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 24 k 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: libcephsqlite x86_64 2:20.2.0-712.g70f8415b.el9 ceph 164 k 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: libconfig x86_64 1.7.2-9.el9 baseos 72 k 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: libgfortran x86_64 11.5.0-14.el9 baseos 794 k 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: libnbd x86_64 1.20.3-4.el9 appstream 164 k 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: liboath x86_64 2.6.12-1.el9 epel 49 k 2026-03-20T11:20:28.598 INFO:teuthology.orchestra.run.vm07.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: libquadmath x86_64 11.5.0-14.el9 baseos 184 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: libradosstriper1 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 250 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: librgw2 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 6.4 M 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: libstoragemgmt x86_64 1.10.1-1.el9 appstream 246 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: libxslt x86_64 1.1.34-12.el9 appstream 233 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: mailcap noarch 2.1.49-5.el9 baseos 33 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: openblas x86_64 0.3.29-1.el9 appstream 42 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: openblas-openmp x86_64 0.3.29-1.el9 appstream 5.3 M 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: parquet-libs x86_64 9.0.0-15.el9 epel 838 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: pciutils x86_64 3.7.0-7.el9 baseos 93 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: perl-Benchmark noarch 1.23-483.el9 appstream 26 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: protobuf x86_64 3.14.0-17.el9 appstream 1.0 M 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: protobuf-compiler x86_64 3.14.0-17.el9 crb 862 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: python3-autocommand noarch 2.2.2-8.el9 epel 29 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: python3-backports-tarfile noarch 1.2.0-1.el9 epel 60 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: python3-ceph-argparse x86_64 2:20.2.0-712.g70f8415b.el9 ceph 45 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: python3-ceph-common x86_64 2:20.2.0-712.g70f8415b.el9 ceph 175 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: python3-cheroot noarch 10.0.1-4.el9 epel 173 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: python3-cryptography x86_64 36.0.1-5.el9 baseos 1.2 M 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: python3-devel x86_64 3.9.25-3.el9 appstream 244 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: python3-google-auth noarch 1:2.45.0-1.el9 epel 254 k 2026-03-20T11:20:28.599 INFO:teuthology.orchestra.run.vm07.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco-context noarch 6.0.1-3.el9 epel 20 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco-text noarch 4.0.0-2.el9 epel 26 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-jinja2 noarch 2.11.3-8.el9 appstream 249 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-kubernetes noarch 1:26.1.0-3.el9 epel 1.0 M 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-libstoragemgmt x86_64 1.10.1-1.el9 appstream 177 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-numpy x86_64 1:1.23.5-2.el9 appstream 6.1 M 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-numpy-f2py x86_64 1:1.23.5-2.el9 appstream 442 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-protobuf noarch 3.14.0-17.el9 appstream 267 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-pyasn1 noarch 0.4.8-7.el9 appstream 157 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-pyasn1-modules noarch 0.4.8-7.el9 appstream 277 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-pyparsing noarch 2.4.7-9.el9 baseos 150 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-requests noarch 2.25.1-10.el9 baseos 126 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-scipy x86_64 1.9.3-2.el9 appstream 19 M 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-typing-extensions noarch 4.15.0-1.el9 epel 86 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-urllib3 noarch 1.26.5-7.el9 baseos 218 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2026-03-20T11:20:28.600 INFO:teuthology.orchestra.run.vm07.stdout: qatlib x86_64 25.08.0-2.el9 appstream 240 k 2026-03-20T11:20:28.601 INFO:teuthology.orchestra.run.vm07.stdout: qatzip-libs x86_64 1.3.1-1.el9 appstream 66 k 2026-03-20T11:20:28.601 INFO:teuthology.orchestra.run.vm07.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2026-03-20T11:20:28.601 INFO:teuthology.orchestra.run.vm07.stdout: socat x86_64 1.7.4.1-8.el9 appstream 303 k 2026-03-20T11:20:28.601 INFO:teuthology.orchestra.run.vm07.stdout: thrift x86_64 0.15.0-4.el9 epel 1.6 M 2026-03-20T11:20:28.601 INFO:teuthology.orchestra.run.vm07.stdout: unzip x86_64 6.0-59.el9 baseos 182 k 2026-03-20T11:20:28.601 INFO:teuthology.orchestra.run.vm07.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2026-03-20T11:20:28.601 INFO:teuthology.orchestra.run.vm07.stdout: zip x86_64 3.0-35.el9 baseos 266 k 2026-03-20T11:20:28.601 INFO:teuthology.orchestra.run.vm07.stdout:Installing weak dependencies: 2026-03-20T11:20:28.601 INFO:teuthology.orchestra.run.vm07.stdout: qatlib-service x86_64 25.08.0-2.el9 appstream 37 k 2026-03-20T11:20:28.601 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:20:28.601 INFO:teuthology.orchestra.run.vm07.stdout:Transaction Summary 2026-03-20T11:20:28.601 INFO:teuthology.orchestra.run.vm07.stdout:====================================================================================== 2026-03-20T11:20:28.601 INFO:teuthology.orchestra.run.vm07.stdout:Install 136 Packages 2026-03-20T11:20:28.601 INFO:teuthology.orchestra.run.vm07.stdout:Upgrade 2 Packages 2026-03-20T11:20:28.601 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:20:28.601 INFO:teuthology.orchestra.run.vm07.stdout:Total download size: 267 M 2026-03-20T11:20:28.601 INFO:teuthology.orchestra.run.vm07.stdout:Downloading Packages: 2026-03-20T11:20:28.885 INFO:teuthology.orchestra.run.vm05.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2026-03-20T11:20:28.885 INFO:teuthology.orchestra.run.vm05.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2026-03-20T11:20:28.921 INFO:teuthology.orchestra.run.vm05.stdout:Dependencies resolved. 2026-03-20T11:20:28.925 INFO:teuthology.orchestra.run.vm05.stdout:====================================================================================== 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: Package Arch Version Repository Size 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout:====================================================================================== 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout:Installing: 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: bzip2 x86_64 1.0.8-11.el9 baseos 55 k 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: ceph x86_64 2:20.2.0-712.g70f8415b.el9 ceph 6.5 k 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: ceph-base x86_64 2:20.2.0-712.g70f8415b.el9 ceph 5.9 M 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: ceph-fuse x86_64 2:20.2.0-712.g70f8415b.el9 ceph 939 k 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: ceph-immutable-object-cache x86_64 2:20.2.0-712.g70f8415b.el9 ceph 154 k 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: ceph-mgr x86_64 2:20.2.0-712.g70f8415b.el9 ceph 962 k 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: ceph-mgr-cephadm noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 173 k 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: ceph-mgr-dashboard noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 11 M 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: ceph-mgr-diskprediction-local noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 7.4 M 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: ceph-mgr-rook noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 50 k 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: ceph-radosgw x86_64 2:20.2.0-712.g70f8415b.el9 ceph 24 M 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: ceph-test x86_64 2:20.2.0-712.g70f8415b.el9 ceph 84 M 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: ceph-volume noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 298 k 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: cephadm noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 1.0 M 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: libcephfs-devel x86_64 2:20.2.0-712.g70f8415b.el9 ceph 34 k 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: libcephfs2 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 866 k 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: librados-devel x86_64 2:20.2.0-712.g70f8415b.el9 ceph 126 k 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: perl-Test-Harness noarch 1:3.42-461.el9 appstream 295 k 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: python3-cephfs x86_64 2:20.2.0-712.g70f8415b.el9 ceph 163 k 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: python3-jmespath noarch 1.0.1-1.el9 appstream 48 k 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: python3-rados x86_64 2:20.2.0-712.g70f8415b.el9 ceph 324 k 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: python3-rbd x86_64 2:20.2.0-712.g70f8415b.el9 ceph 304 k 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: python3-rgw x86_64 2:20.2.0-712.g70f8415b.el9 ceph 99 k 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: python3-xmltodict noarch 0.12.0-15.el9 epel 22 k 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: rbd-fuse x86_64 2:20.2.0-712.g70f8415b.el9 ceph 91 k 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: rbd-mirror x86_64 2:20.2.0-712.g70f8415b.el9 ceph 2.9 M 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: rbd-nbd x86_64 2:20.2.0-712.g70f8415b.el9 ceph 180 k 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: s3cmd noarch 2.4.0-1.el9 epel 206 k 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout:Upgrading: 2026-03-20T11:20:28.926 INFO:teuthology.orchestra.run.vm05.stdout: librados2 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 3.5 M 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: librbd1 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 2.8 M 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout:Installing dependencies: 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: abseil-cpp x86_64 20211102.0-4.el9 epel 551 k 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: boost-program-options x86_64 1.75.0-13.el9 appstream 104 k 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: ceph-common x86_64 2:20.2.0-712.g70f8415b.el9 ceph 24 M 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: ceph-grafana-dashboards noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 43 k 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: ceph-mds x86_64 2:20.2.0-712.g70f8415b.el9 ceph 2.3 M 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: ceph-mgr-modules-core noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 290 k 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: ceph-mon x86_64 2:20.2.0-712.g70f8415b.el9 ceph 5.0 M 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: ceph-osd x86_64 2:20.2.0-712.g70f8415b.el9 ceph 17 M 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: ceph-prometheus-alerts noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 17 k 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: ceph-selinux x86_64 2:20.2.0-712.g70f8415b.el9 ceph 25 k 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: cryptsetup x86_64 2.8.1-3.el9 baseos 351 k 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: flexiblas x86_64 3.0.4-9.el9 appstream 30 k 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: flexiblas-netlib x86_64 3.0.4-9.el9 appstream 3.0 M 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: flexiblas-openblas-openmp x86_64 3.0.4-9.el9 appstream 15 k 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: fuse x86_64 2.9.9-17.el9 baseos 80 k 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: ledmon-libs x86_64 1.1.0-3.el9 baseos 40 k 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: libarrow x86_64 9.0.0-15.el9 epel 4.4 M 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: libarrow-doc noarch 9.0.0-15.el9 epel 25 k 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: libcephfs-proxy2 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 24 k 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: libcephsqlite x86_64 2:20.2.0-712.g70f8415b.el9 ceph 164 k 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: libconfig x86_64 1.7.2-9.el9 baseos 72 k 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: libgfortran x86_64 11.5.0-14.el9 baseos 794 k 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: libnbd x86_64 1.20.3-4.el9 appstream 164 k 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: liboath x86_64 2.6.12-1.el9 epel 49 k 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: libquadmath x86_64 11.5.0-14.el9 baseos 184 k 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: libradosstriper1 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 250 k 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: librgw2 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 6.4 M 2026-03-20T11:20:28.927 INFO:teuthology.orchestra.run.vm05.stdout: libstoragemgmt x86_64 1.10.1-1.el9 appstream 246 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: libxslt x86_64 1.1.34-12.el9 appstream 233 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: mailcap noarch 2.1.49-5.el9 baseos 33 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: openblas x86_64 0.3.29-1.el9 appstream 42 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: openblas-openmp x86_64 0.3.29-1.el9 appstream 5.3 M 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: parquet-libs x86_64 9.0.0-15.el9 epel 838 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: pciutils x86_64 3.7.0-7.el9 baseos 93 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: perl-Benchmark noarch 1.23-483.el9 appstream 26 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: protobuf x86_64 3.14.0-17.el9 appstream 1.0 M 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: protobuf-compiler x86_64 3.14.0-17.el9 crb 862 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: python3-autocommand noarch 2.2.2-8.el9 epel 29 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: python3-backports-tarfile noarch 1.2.0-1.el9 epel 60 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: python3-ceph-argparse x86_64 2:20.2.0-712.g70f8415b.el9 ceph 45 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: python3-ceph-common x86_64 2:20.2.0-712.g70f8415b.el9 ceph 175 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: python3-cheroot noarch 10.0.1-4.el9 epel 173 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: python3-cryptography x86_64 36.0.1-5.el9 baseos 1.2 M 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: python3-devel x86_64 3.9.25-3.el9 appstream 244 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: python3-google-auth noarch 1:2.45.0-1.el9 epel 254 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: python3-jaraco-context noarch 6.0.1-3.el9 epel 20 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: python3-jaraco-text noarch 4.0.0-2.el9 epel 26 k 2026-03-20T11:20:28.928 INFO:teuthology.orchestra.run.vm05.stdout: python3-jinja2 noarch 2.11.3-8.el9 appstream 249 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: python3-kubernetes noarch 1:26.1.0-3.el9 epel 1.0 M 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: python3-libstoragemgmt x86_64 1.10.1-1.el9 appstream 177 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: python3-numpy x86_64 1:1.23.5-2.el9 appstream 6.1 M 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: python3-numpy-f2py x86_64 1:1.23.5-2.el9 appstream 442 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: python3-protobuf noarch 3.14.0-17.el9 appstream 267 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: python3-pyasn1 noarch 0.4.8-7.el9 appstream 157 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: python3-pyasn1-modules noarch 0.4.8-7.el9 appstream 277 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: python3-pyparsing noarch 2.4.7-9.el9 baseos 150 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: python3-requests noarch 2.25.1-10.el9 baseos 126 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: python3-scipy x86_64 1.9.3-2.el9 appstream 19 M 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: python3-typing-extensions noarch 4.15.0-1.el9 epel 86 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: python3-urllib3 noarch 1.26.5-7.el9 baseos 218 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: qatlib x86_64 25.08.0-2.el9 appstream 240 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: qatzip-libs x86_64 1.3.1-1.el9 appstream 66 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: socat x86_64 1.7.4.1-8.el9 appstream 303 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: thrift x86_64 0.15.0-4.el9 epel 1.6 M 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: unzip x86_64 6.0-59.el9 baseos 182 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2026-03-20T11:20:28.929 INFO:teuthology.orchestra.run.vm05.stdout: zip x86_64 3.0-35.el9 baseos 266 k 2026-03-20T11:20:28.930 INFO:teuthology.orchestra.run.vm05.stdout:Installing weak dependencies: 2026-03-20T11:20:28.930 INFO:teuthology.orchestra.run.vm05.stdout: qatlib-service x86_64 25.08.0-2.el9 appstream 37 k 2026-03-20T11:20:28.930 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:20:28.930 INFO:teuthology.orchestra.run.vm05.stdout:Transaction Summary 2026-03-20T11:20:28.930 INFO:teuthology.orchestra.run.vm05.stdout:====================================================================================== 2026-03-20T11:20:28.930 INFO:teuthology.orchestra.run.vm05.stdout:Install 136 Packages 2026-03-20T11:20:28.930 INFO:teuthology.orchestra.run.vm05.stdout:Upgrade 2 Packages 2026-03-20T11:20:28.930 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:20:28.930 INFO:teuthology.orchestra.run.vm05.stdout:Total download size: 267 M 2026-03-20T11:20:28.930 INFO:teuthology.orchestra.run.vm05.stdout:Downloading Packages: 2026-03-20T11:20:29.546 INFO:teuthology.orchestra.run.vm02.stdout:(1/138): ceph-20.2.0-712.g70f8415b.el9.x86_64.r 14 kB/s | 6.5 kB 00:00 2026-03-20T11:20:30.166 INFO:teuthology.orchestra.run.vm05.stdout:(1/138): ceph-20.2.0-712.g70f8415b.el9.x86_64.r 14 kB/s | 6.5 kB 00:00 2026-03-20T11:20:30.349 INFO:teuthology.orchestra.run.vm07.stdout:(1/138): ceph-20.2.0-712.g70f8415b.el9.x86_64.r 14 kB/s | 6.5 kB 00:00 2026-03-20T11:20:30.371 INFO:teuthology.orchestra.run.vm02.stdout:(2/138): ceph-fuse-20.2.0-712.g70f8415b.el9.x86 1.1 MB/s | 939 kB 00:00 2026-03-20T11:20:30.489 INFO:teuthology.orchestra.run.vm02.stdout:(3/138): ceph-immutable-object-cache-20.2.0-712 1.3 MB/s | 154 kB 00:00 2026-03-20T11:20:30.825 INFO:teuthology.orchestra.run.vm02.stdout:(4/138): ceph-base-20.2.0-712.g70f8415b.el9.x86 3.4 MB/s | 5.9 MB 00:01 2026-03-20T11:20:30.855 INFO:teuthology.orchestra.run.vm02.stdout:(5/138): ceph-mds-20.2.0-712.g70f8415b.el9.x86_ 6.4 MB/s | 2.3 MB 00:00 2026-03-20T11:20:30.953 INFO:teuthology.orchestra.run.vm05.stdout:(2/138): ceph-fuse-20.2.0-712.g70f8415b.el9.x86 1.2 MB/s | 939 kB 00:00 2026-03-20T11:20:30.954 INFO:teuthology.orchestra.run.vm02.stdout:(6/138): ceph-mgr-20.2.0-712.g70f8415b.el9.x86_ 7.4 MB/s | 962 kB 00:00 2026-03-20T11:20:31.081 INFO:teuthology.orchestra.run.vm05.stdout:(3/138): ceph-immutable-object-cache-20.2.0-712 1.2 MB/s | 154 kB 00:00 2026-03-20T11:20:31.186 INFO:teuthology.orchestra.run.vm07.stdout:(2/138): ceph-fuse-20.2.0-712.g70f8415b.el9.x86 1.1 MB/s | 939 kB 00:00 2026-03-20T11:20:31.321 INFO:teuthology.orchestra.run.vm07.stdout:(3/138): ceph-immutable-object-cache-20.2.0-712 1.1 MB/s | 154 kB 00:00 2026-03-20T11:20:31.505 INFO:teuthology.orchestra.run.vm05.stdout:(4/138): ceph-mds-20.2.0-712.g70f8415b.el9.x86_ 5.5 MB/s | 2.3 MB 00:00 2026-03-20T11:20:31.533 INFO:teuthology.orchestra.run.vm08.stdout:lab-extras 65 kB/s | 50 kB 00:00 2026-03-20T11:20:31.620 INFO:teuthology.orchestra.run.vm05.stdout:(5/138): ceph-base-20.2.0-712.g70f8415b.el9.x86 3.1 MB/s | 5.9 MB 00:01 2026-03-20T11:20:31.667 INFO:teuthology.orchestra.run.vm02.stdout:(7/138): ceph-mon-20.2.0-712.g70f8415b.el9.x86_ 6.2 MB/s | 5.0 MB 00:00 2026-03-20T11:20:31.804 INFO:teuthology.orchestra.run.vm07.stdout:(4/138): ceph-mds-20.2.0-712.g70f8415b.el9.x86_ 4.9 MB/s | 2.3 MB 00:00 2026-03-20T11:20:31.825 INFO:teuthology.orchestra.run.vm05.stdout:(6/138): ceph-mgr-20.2.0-712.g70f8415b.el9.x86_ 2.9 MB/s | 962 kB 00:00 2026-03-20T11:20:31.870 INFO:teuthology.orchestra.run.vm07.stdout:(5/138): ceph-base-20.2.0-712.g70f8415b.el9.x86 2.9 MB/s | 5.9 MB 00:02 2026-03-20T11:20:32.106 INFO:teuthology.orchestra.run.vm07.stdout:(6/138): ceph-mgr-20.2.0-712.g70f8415b.el9.x86_ 3.1 MB/s | 962 kB 00:00 2026-03-20T11:20:32.395 INFO:teuthology.orchestra.run.vm05.stdout:(7/138): ceph-mon-20.2.0-712.g70f8415b.el9.x86_ 6.5 MB/s | 5.0 MB 00:00 2026-03-20T11:20:32.967 INFO:teuthology.orchestra.run.vm07.stdout:(7/138): ceph-mon-20.2.0-712.g70f8415b.el9.x86_ 4.6 MB/s | 5.0 MB 00:01 2026-03-20T11:20:33.057 INFO:teuthology.orchestra.run.vm08.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2026-03-20T11:20:33.058 INFO:teuthology.orchestra.run.vm08.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2026-03-20T11:20:33.097 INFO:teuthology.orchestra.run.vm08.stdout:Dependencies resolved. 2026-03-20T11:20:33.101 INFO:teuthology.orchestra.run.vm08.stdout:====================================================================================== 2026-03-20T11:20:33.101 INFO:teuthology.orchestra.run.vm08.stdout: Package Arch Version Repository Size 2026-03-20T11:20:33.101 INFO:teuthology.orchestra.run.vm08.stdout:====================================================================================== 2026-03-20T11:20:33.101 INFO:teuthology.orchestra.run.vm08.stdout:Installing: 2026-03-20T11:20:33.101 INFO:teuthology.orchestra.run.vm08.stdout: bzip2 x86_64 1.0.8-11.el9 baseos 55 k 2026-03-20T11:20:33.101 INFO:teuthology.orchestra.run.vm08.stdout: ceph x86_64 2:20.2.0-712.g70f8415b.el9 ceph 6.5 k 2026-03-20T11:20:33.101 INFO:teuthology.orchestra.run.vm08.stdout: ceph-base x86_64 2:20.2.0-712.g70f8415b.el9 ceph 5.9 M 2026-03-20T11:20:33.101 INFO:teuthology.orchestra.run.vm08.stdout: ceph-fuse x86_64 2:20.2.0-712.g70f8415b.el9 ceph 939 k 2026-03-20T11:20:33.101 INFO:teuthology.orchestra.run.vm08.stdout: ceph-immutable-object-cache x86_64 2:20.2.0-712.g70f8415b.el9 ceph 154 k 2026-03-20T11:20:33.101 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr x86_64 2:20.2.0-712.g70f8415b.el9 ceph 962 k 2026-03-20T11:20:33.101 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-cephadm noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 173 k 2026-03-20T11:20:33.101 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-dashboard noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 11 M 2026-03-20T11:20:33.101 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-diskprediction-local noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 7.4 M 2026-03-20T11:20:33.101 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-rook noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 50 k 2026-03-20T11:20:33.101 INFO:teuthology.orchestra.run.vm08.stdout: ceph-radosgw x86_64 2:20.2.0-712.g70f8415b.el9 ceph 24 M 2026-03-20T11:20:33.101 INFO:teuthology.orchestra.run.vm08.stdout: ceph-test x86_64 2:20.2.0-712.g70f8415b.el9 ceph 84 M 2026-03-20T11:20:33.101 INFO:teuthology.orchestra.run.vm08.stdout: ceph-volume noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 298 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: cephadm noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 1.0 M 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: libcephfs-devel x86_64 2:20.2.0-712.g70f8415b.el9 ceph 34 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: libcephfs2 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 866 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: librados-devel x86_64 2:20.2.0-712.g70f8415b.el9 ceph 126 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: perl-Test-Harness noarch 1:3.42-461.el9 appstream 295 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: python3-cephfs x86_64 2:20.2.0-712.g70f8415b.el9 ceph 163 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: python3-jmespath noarch 1.0.1-1.el9 appstream 48 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: python3-rados x86_64 2:20.2.0-712.g70f8415b.el9 ceph 324 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: python3-rbd x86_64 2:20.2.0-712.g70f8415b.el9 ceph 304 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: python3-rgw x86_64 2:20.2.0-712.g70f8415b.el9 ceph 99 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: python3-xmltodict noarch 0.12.0-15.el9 epel 22 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: rbd-fuse x86_64 2:20.2.0-712.g70f8415b.el9 ceph 91 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: rbd-mirror x86_64 2:20.2.0-712.g70f8415b.el9 ceph 2.9 M 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: rbd-nbd x86_64 2:20.2.0-712.g70f8415b.el9 ceph 180 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: s3cmd noarch 2.4.0-1.el9 epel 206 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout:Upgrading: 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: librados2 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 3.5 M 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: librbd1 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 2.8 M 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout:Installing dependencies: 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: abseil-cpp x86_64 20211102.0-4.el9 epel 551 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: boost-program-options x86_64 1.75.0-13.el9 appstream 104 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: ceph-common x86_64 2:20.2.0-712.g70f8415b.el9 ceph 24 M 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: ceph-grafana-dashboards noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 43 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mds x86_64 2:20.2.0-712.g70f8415b.el9 ceph 2.3 M 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-modules-core noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 290 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mon x86_64 2:20.2.0-712.g70f8415b.el9 ceph 5.0 M 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: ceph-osd x86_64 2:20.2.0-712.g70f8415b.el9 ceph 17 M 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: ceph-prometheus-alerts noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 17 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: ceph-selinux x86_64 2:20.2.0-712.g70f8415b.el9 ceph 25 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: cryptsetup x86_64 2.8.1-3.el9 baseos 351 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: flexiblas x86_64 3.0.4-9.el9 appstream 30 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: flexiblas-netlib x86_64 3.0.4-9.el9 appstream 3.0 M 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: flexiblas-openblas-openmp x86_64 3.0.4-9.el9 appstream 15 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: fuse x86_64 2.9.9-17.el9 baseos 80 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: ledmon-libs x86_64 1.1.0-3.el9 baseos 40 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: libarrow x86_64 9.0.0-15.el9 epel 4.4 M 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: libarrow-doc noarch 9.0.0-15.el9 epel 25 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: libcephfs-proxy2 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 24 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: libcephsqlite x86_64 2:20.2.0-712.g70f8415b.el9 ceph 164 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: libconfig x86_64 1.7.2-9.el9 baseos 72 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: libgfortran x86_64 11.5.0-14.el9 baseos 794 k 2026-03-20T11:20:33.102 INFO:teuthology.orchestra.run.vm08.stdout: libnbd x86_64 1.20.3-4.el9 appstream 164 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: liboath x86_64 2.6.12-1.el9 epel 49 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: libquadmath x86_64 11.5.0-14.el9 baseos 184 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: libradosstriper1 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 250 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: librgw2 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 6.4 M 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: libstoragemgmt x86_64 1.10.1-1.el9 appstream 246 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: libxslt x86_64 1.1.34-12.el9 appstream 233 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: mailcap noarch 2.1.49-5.el9 baseos 33 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: openblas x86_64 0.3.29-1.el9 appstream 42 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: openblas-openmp x86_64 0.3.29-1.el9 appstream 5.3 M 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: parquet-libs x86_64 9.0.0-15.el9 epel 838 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: pciutils x86_64 3.7.0-7.el9 baseos 93 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: perl-Benchmark noarch 1.23-483.el9 appstream 26 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: protobuf x86_64 3.14.0-17.el9 appstream 1.0 M 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: protobuf-compiler x86_64 3.14.0-17.el9 crb 862 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: python3-autocommand noarch 2.2.2-8.el9 epel 29 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: python3-backports-tarfile noarch 1.2.0-1.el9 epel 60 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: python3-ceph-argparse x86_64 2:20.2.0-712.g70f8415b.el9 ceph 45 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: python3-ceph-common x86_64 2:20.2.0-712.g70f8415b.el9 ceph 175 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: python3-cheroot noarch 10.0.1-4.el9 epel 173 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: python3-cryptography x86_64 36.0.1-5.el9 baseos 1.2 M 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: python3-devel x86_64 3.9.25-3.el9 appstream 244 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: python3-google-auth noarch 1:2.45.0-1.el9 epel 254 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-context noarch 6.0.1-3.el9 epel 20 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2026-03-20T11:20:33.103 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-text noarch 4.0.0-2.el9 epel 26 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-jinja2 noarch 2.11.3-8.el9 appstream 249 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-kubernetes noarch 1:26.1.0-3.el9 epel 1.0 M 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-libstoragemgmt x86_64 1.10.1-1.el9 appstream 177 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-numpy x86_64 1:1.23.5-2.el9 appstream 6.1 M 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-numpy-f2py x86_64 1:1.23.5-2.el9 appstream 442 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-protobuf noarch 3.14.0-17.el9 appstream 267 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-pyasn1 noarch 0.4.8-7.el9 appstream 157 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-pyasn1-modules noarch 0.4.8-7.el9 appstream 277 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-pyparsing noarch 2.4.7-9.el9 baseos 150 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-requests noarch 2.25.1-10.el9 baseos 126 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-scipy x86_64 1.9.3-2.el9 appstream 19 M 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-typing-extensions noarch 4.15.0-1.el9 epel 86 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-urllib3 noarch 1.26.5-7.el9 baseos 218 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: qatlib x86_64 25.08.0-2.el9 appstream 240 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: qatzip-libs x86_64 1.3.1-1.el9 appstream 66 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: socat x86_64 1.7.4.1-8.el9 appstream 303 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: thrift x86_64 0.15.0-4.el9 epel 1.6 M 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: unzip x86_64 6.0-59.el9 baseos 182 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout: zip x86_64 3.0-35.el9 baseos 266 k 2026-03-20T11:20:33.104 INFO:teuthology.orchestra.run.vm08.stdout:Installing weak dependencies: 2026-03-20T11:20:33.105 INFO:teuthology.orchestra.run.vm08.stdout: qatlib-service x86_64 25.08.0-2.el9 appstream 37 k 2026-03-20T11:20:33.105 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:20:33.105 INFO:teuthology.orchestra.run.vm08.stdout:Transaction Summary 2026-03-20T11:20:33.105 INFO:teuthology.orchestra.run.vm08.stdout:====================================================================================== 2026-03-20T11:20:33.105 INFO:teuthology.orchestra.run.vm08.stdout:Install 136 Packages 2026-03-20T11:20:33.105 INFO:teuthology.orchestra.run.vm08.stdout:Upgrade 2 Packages 2026-03-20T11:20:33.105 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:20:33.105 INFO:teuthology.orchestra.run.vm08.stdout:Total download size: 267 M 2026-03-20T11:20:33.105 INFO:teuthology.orchestra.run.vm08.stdout:Downloading Packages: 2026-03-20T11:20:33.350 INFO:teuthology.orchestra.run.vm02.stdout:(8/138): ceph-common-20.2.0-712.g70f8415b.el9.x 5.6 MB/s | 24 MB 00:04 2026-03-20T11:20:33.470 INFO:teuthology.orchestra.run.vm02.stdout:(9/138): ceph-osd-20.2.0-712.g70f8415b.el9.x86_ 6.7 MB/s | 17 MB 00:02 2026-03-20T11:20:33.493 INFO:teuthology.orchestra.run.vm02.stdout:(10/138): ceph-selinux-20.2.0-712.g70f8415b.el9 177 kB/s | 25 kB 00:00 2026-03-20T11:20:33.576 INFO:teuthology.orchestra.run.vm05.stdout:(8/138): ceph-common-20.2.0-712.g70f8415b.el9.x 6.2 MB/s | 24 MB 00:03 2026-03-20T11:20:33.625 INFO:teuthology.orchestra.run.vm02.stdout:(11/138): libcephfs-devel-20.2.0-712.g70f8415b. 260 kB/s | 34 kB 00:00 2026-03-20T11:20:33.704 INFO:teuthology.orchestra.run.vm05.stdout:(9/138): ceph-selinux-20.2.0-712.g70f8415b.el9. 198 kB/s | 25 kB 00:00 2026-03-20T11:20:33.704 INFO:teuthology.orchestra.run.vm07.stdout:(8/138): ceph-common-20.2.0-712.g70f8415b.el9.x 6.2 MB/s | 24 MB 00:03 2026-03-20T11:20:33.739 INFO:teuthology.orchestra.run.vm02.stdout:(12/138): libcephfs-proxy2-20.2.0-712.g70f8415b 211 kB/s | 24 kB 00:00 2026-03-20T11:20:33.868 INFO:teuthology.orchestra.run.vm07.stdout:(9/138): ceph-selinux-20.2.0-712.g70f8415b.el9. 154 kB/s | 25 kB 00:00 2026-03-20T11:20:34.276 INFO:teuthology.orchestra.run.vm07.stdout:(10/138): ceph-osd-20.2.0-712.g70f8415b.el9.x86 7.8 MB/s | 17 MB 00:02 2026-03-20T11:20:34.328 INFO:teuthology.orchestra.run.vm02.stdout:(13/138): libcephfs2-20.2.0-712.g70f8415b.el9.x 1.4 MB/s | 866 kB 00:00 2026-03-20T11:20:34.400 INFO:teuthology.orchestra.run.vm07.stdout:(11/138): libcephfs-devel-20.2.0-712.g70f8415b. 278 kB/s | 34 kB 00:00 2026-03-20T11:20:34.460 INFO:teuthology.orchestra.run.vm02.stdout:(14/138): libcephsqlite-20.2.0-712.g70f8415b.el 1.2 MB/s | 164 kB 00:00 2026-03-20T11:20:34.524 INFO:teuthology.orchestra.run.vm07.stdout:(12/138): libcephfs-proxy2-20.2.0-712.g70f8415b 195 kB/s | 24 kB 00:00 2026-03-20T11:20:34.583 INFO:teuthology.orchestra.run.vm02.stdout:(15/138): librados-devel-20.2.0-712.g70f8415b.e 1.0 MB/s | 126 kB 00:00 2026-03-20T11:20:34.721 INFO:teuthology.orchestra.run.vm02.stdout:(16/138): libradosstriper1-20.2.0-712.g70f8415b 1.8 MB/s | 250 kB 00:00 2026-03-20T11:20:34.795 INFO:teuthology.orchestra.run.vm07.stdout:(13/138): libcephfs2-20.2.0-712.g70f8415b.el9.x 3.1 MB/s | 866 kB 00:00 2026-03-20T11:20:34.855 INFO:teuthology.orchestra.run.vm08.stdout:(1/138): ceph-20.2.0-712.g70f8415b.el9.x86_64.r 13 kB/s | 6.5 kB 00:00 2026-03-20T11:20:34.933 INFO:teuthology.orchestra.run.vm07.stdout:(14/138): libcephsqlite-20.2.0-712.g70f8415b.el 1.2 MB/s | 164 kB 00:00 2026-03-20T11:20:35.017 INFO:teuthology.orchestra.run.vm05.stdout:(10/138): ceph-osd-20.2.0-712.g70f8415b.el9.x86 5.3 MB/s | 17 MB 00:03 2026-03-20T11:20:35.109 INFO:teuthology.orchestra.run.vm07.stdout:(15/138): librados-devel-20.2.0-712.g70f8415b.e 717 kB/s | 126 kB 00:00 2026-03-20T11:20:35.166 INFO:teuthology.orchestra.run.vm05.stdout:(11/138): libcephfs-devel-20.2.0-712.g70f8415b. 230 kB/s | 34 kB 00:00 2026-03-20T11:20:35.241 INFO:teuthology.orchestra.run.vm07.stdout:(16/138): libradosstriper1-20.2.0-712.g70f8415b 1.8 MB/s | 250 kB 00:00 2026-03-20T11:20:35.294 INFO:teuthology.orchestra.run.vm05.stdout:(12/138): libcephfs-proxy2-20.2.0-712.g70f8415b 190 kB/s | 24 kB 00:00 2026-03-20T11:20:35.601 INFO:teuthology.orchestra.run.vm05.stdout:(13/138): libcephfs2-20.2.0-712.g70f8415b.el9.x 2.8 MB/s | 866 kB 00:00 2026-03-20T11:20:35.758 INFO:teuthology.orchestra.run.vm05.stdout:(14/138): libcephsqlite-20.2.0-712.g70f8415b.el 1.0 MB/s | 164 kB 00:00 2026-03-20T11:20:35.833 INFO:teuthology.orchestra.run.vm05.stdout:(15/138): ceph-radosgw-20.2.0-712.g70f8415b.el9 6.9 MB/s | 24 MB 00:03 2026-03-20T11:20:35.901 INFO:teuthology.orchestra.run.vm05.stdout:(16/138): librados-devel-20.2.0-712.g70f8415b.e 883 kB/s | 126 kB 00:00 2026-03-20T11:20:35.953 INFO:teuthology.orchestra.run.vm08.stdout:(2/138): ceph-fuse-20.2.0-712.g70f8415b.el9.x86 855 kB/s | 939 kB 00:01 2026-03-20T11:20:35.978 INFO:teuthology.orchestra.run.vm05.stdout:(17/138): libradosstriper1-20.2.0-712.g70f8415b 1.7 MB/s | 250 kB 00:00 2026-03-20T11:20:36.112 INFO:teuthology.orchestra.run.vm08.stdout:(3/138): ceph-immutable-object-cache-20.2.0-712 974 kB/s | 154 kB 00:00 2026-03-20T11:20:36.129 INFO:teuthology.orchestra.run.vm05.stdout:(18/138): python3-ceph-argparse-20.2.0-712.g70f 299 kB/s | 45 kB 00:00 2026-03-20T11:20:36.179 INFO:teuthology.orchestra.run.vm02.stdout:(17/138): ceph-radosgw-20.2.0-712.g70f8415b.el9 5.2 MB/s | 24 MB 00:04 2026-03-20T11:20:36.279 INFO:teuthology.orchestra.run.vm05.stdout:(19/138): python3-ceph-common-20.2.0-712.g70f84 1.1 MB/s | 175 kB 00:00 2026-03-20T11:20:36.330 INFO:teuthology.orchestra.run.vm02.stdout:(18/138): python3-ceph-argparse-20.2.0-712.g70f 299 kB/s | 45 kB 00:00 2026-03-20T11:20:36.431 INFO:teuthology.orchestra.run.vm05.stdout:(20/138): python3-cephfs-20.2.0-712.g70f8415b.e 1.0 MB/s | 163 kB 00:00 2026-03-20T11:20:36.441 INFO:teuthology.orchestra.run.vm07.stdout:(17/138): librgw2-20.2.0-712.g70f8415b.el9.x86_ 5.3 MB/s | 6.4 MB 00:01 2026-03-20T11:20:36.474 INFO:teuthology.orchestra.run.vm02.stdout:(19/138): python3-ceph-common-20.2.0-712.g70f84 1.2 MB/s | 175 kB 00:00 2026-03-20T11:20:36.581 INFO:teuthology.orchestra.run.vm05.stdout:(21/138): python3-rados-20.2.0-712.g70f8415b.el 2.1 MB/s | 324 kB 00:00 2026-03-20T11:20:36.586 INFO:teuthology.orchestra.run.vm07.stdout:(18/138): python3-ceph-argparse-20.2.0-712.g70f 313 kB/s | 45 kB 00:00 2026-03-20T11:20:36.621 INFO:teuthology.orchestra.run.vm02.stdout:(20/138): python3-cephfs-20.2.0-712.g70f8415b.e 1.1 MB/s | 163 kB 00:00 2026-03-20T11:20:36.659 INFO:teuthology.orchestra.run.vm02.stdout:(21/138): librgw2-20.2.0-712.g70f8415b.el9.x86_ 3.3 MB/s | 6.4 MB 00:01 2026-03-20T11:20:36.698 INFO:teuthology.orchestra.run.vm07.stdout:(19/138): ceph-radosgw-20.2.0-712.g70f8415b.el9 6.3 MB/s | 24 MB 00:03 2026-03-20T11:20:36.706 INFO:teuthology.orchestra.run.vm05.stdout:(22/138): python3-rbd-20.2.0-712.g70f8415b.el9. 2.4 MB/s | 304 kB 00:00 2026-03-20T11:20:36.712 INFO:teuthology.orchestra.run.vm07.stdout:(20/138): python3-ceph-common-20.2.0-712.g70f84 1.4 MB/s | 175 kB 00:00 2026-03-20T11:20:36.753 INFO:teuthology.orchestra.run.vm02.stdout:(22/138): python3-rados-20.2.0-712.g70f8415b.el 2.4 MB/s | 324 kB 00:00 2026-03-20T11:20:36.776 INFO:teuthology.orchestra.run.vm02.stdout:(23/138): python3-rbd-20.2.0-712.g70f8415b.el9. 2.6 MB/s | 304 kB 00:00 2026-03-20T11:20:36.823 INFO:teuthology.orchestra.run.vm07.stdout:(21/138): python3-cephfs-20.2.0-712.g70f8415b.e 1.3 MB/s | 163 kB 00:00 2026-03-20T11:20:36.835 INFO:teuthology.orchestra.run.vm05.stdout:(23/138): python3-rgw-20.2.0-712.g70f8415b.el9. 768 kB/s | 99 kB 00:00 2026-03-20T11:20:36.847 INFO:teuthology.orchestra.run.vm07.stdout:(22/138): python3-rados-20.2.0-712.g70f8415b.el 2.4 MB/s | 324 kB 00:00 2026-03-20T11:20:36.881 INFO:teuthology.orchestra.run.vm02.stdout:(24/138): python3-rgw-20.2.0-712.g70f8415b.el9. 773 kB/s | 99 kB 00:00 2026-03-20T11:20:36.891 INFO:teuthology.orchestra.run.vm02.stdout:(25/138): rbd-fuse-20.2.0-712.g70f8415b.el9.x86 791 kB/s | 91 kB 00:00 2026-03-20T11:20:36.916 INFO:teuthology.orchestra.run.vm08.stdout:(4/138): ceph-base-20.2.0-712.g70f8415b.el9.x86 2.3 MB/s | 5.9 MB 00:02 2026-03-20T11:20:36.944 INFO:teuthology.orchestra.run.vm07.stdout:(23/138): python3-rbd-20.2.0-712.g70f8415b.el9. 2.5 MB/s | 304 kB 00:00 2026-03-20T11:20:36.959 INFO:teuthology.orchestra.run.vm05.stdout:(24/138): rbd-fuse-20.2.0-712.g70f8415b.el9.x86 738 kB/s | 91 kB 00:00 2026-03-20T11:20:36.968 INFO:teuthology.orchestra.run.vm07.stdout:(24/138): python3-rgw-20.2.0-712.g70f8415b.el9. 824 kB/s | 99 kB 00:00 2026-03-20T11:20:37.011 INFO:teuthology.orchestra.run.vm02.stdout:(26/138): rbd-nbd-20.2.0-712.g70f8415b.el9.x86_ 1.5 MB/s | 180 kB 00:00 2026-03-20T11:20:37.072 INFO:teuthology.orchestra.run.vm07.stdout:(25/138): rbd-fuse-20.2.0-712.g70f8415b.el9.x86 712 kB/s | 91 kB 00:00 2026-03-20T11:20:37.167 INFO:teuthology.orchestra.run.vm02.stdout:(27/138): ceph-grafana-dashboards-20.2.0-712.g7 277 kB/s | 43 kB 00:00 2026-03-20T11:20:37.195 INFO:teuthology.orchestra.run.vm08.stdout:(5/138): ceph-mgr-20.2.0-712.g70f8415b.el9.x86_ 3.4 MB/s | 962 kB 00:00 2026-03-20T11:20:37.228 INFO:teuthology.orchestra.run.vm07.stdout:(26/138): rbd-nbd-20.2.0-712.g70f8415b.el9.x86_ 1.1 MB/s | 180 kB 00:00 2026-03-20T11:20:37.253 INFO:teuthology.orchestra.run.vm05.stdout:(25/138): librgw2-20.2.0-712.g70f8415b.el9.x86_ 4.7 MB/s | 6.4 MB 00:01 2026-03-20T11:20:37.315 INFO:teuthology.orchestra.run.vm02.stdout:(28/138): ceph-mgr-cephadm-20.2.0-712.g70f8415b 1.1 MB/s | 173 kB 00:00 2026-03-20T11:20:37.366 INFO:teuthology.orchestra.run.vm08.stdout:(6/138): ceph-mds-20.2.0-712.g70f8415b.el9.x86_ 1.9 MB/s | 2.3 MB 00:01 2026-03-20T11:20:37.382 INFO:teuthology.orchestra.run.vm07.stdout:(27/138): ceph-grafana-dashboards-20.2.0-712.g7 282 kB/s | 43 kB 00:00 2026-03-20T11:20:37.403 INFO:teuthology.orchestra.run.vm05.stdout:(26/138): rbd-nbd-20.2.0-712.g70f8415b.el9.x86_ 1.2 MB/s | 180 kB 00:00 2026-03-20T11:20:37.417 INFO:teuthology.orchestra.run.vm05.stdout:(27/138): rbd-mirror-20.2.0-712.g70f8415b.el9.x 6.4 MB/s | 2.9 MB 00:00 2026-03-20T11:20:37.439 INFO:teuthology.orchestra.run.vm07.stdout:(28/138): rbd-mirror-20.2.0-712.g70f8415b.el9.x 6.2 MB/s | 2.9 MB 00:00 2026-03-20T11:20:37.533 INFO:teuthology.orchestra.run.vm07.stdout:(29/138): ceph-mgr-cephadm-20.2.0-712.g70f8415b 1.1 MB/s | 173 kB 00:00 2026-03-20T11:20:37.542 INFO:teuthology.orchestra.run.vm05.stdout:(28/138): ceph-grafana-dashboards-20.2.0-712.g7 313 kB/s | 43 kB 00:00 2026-03-20T11:20:37.558 INFO:teuthology.orchestra.run.vm05.stdout:(29/138): ceph-mgr-cephadm-20.2.0-712.g70f8415b 1.2 MB/s | 173 kB 00:00 2026-03-20T11:20:37.597 INFO:teuthology.orchestra.run.vm02.stdout:(29/138): rbd-mirror-20.2.0-712.g70f8415b.el9.x 4.1 MB/s | 2.9 MB 00:00 2026-03-20T11:20:38.428 INFO:teuthology.orchestra.run.vm08.stdout:(7/138): ceph-mon-20.2.0-712.g70f8415b.el9.x86_ 4.1 MB/s | 5.0 MB 00:01 2026-03-20T11:20:38.874 INFO:teuthology.orchestra.run.vm07.stdout:(30/138): ceph-mgr-dashboard-20.2.0-712.g70f841 7.4 MB/s | 11 MB 00:01 2026-03-20T11:20:38.997 INFO:teuthology.orchestra.run.vm05.stdout:(30/138): ceph-mgr-diskprediction-local-20.2.0- 5.2 MB/s | 7.4 MB 00:01 2026-03-20T11:20:39.018 INFO:teuthology.orchestra.run.vm07.stdout:(31/138): ceph-mgr-diskprediction-local-20.2.0- 5.0 MB/s | 7.4 MB 00:01 2026-03-20T11:20:39.021 INFO:teuthology.orchestra.run.vm07.stdout:(32/138): ceph-mgr-modules-core-20.2.0-712.g70f 1.9 MB/s | 290 kB 00:00 2026-03-20T11:20:39.131 INFO:teuthology.orchestra.run.vm05.stdout:(31/138): ceph-mgr-modules-core-20.2.0-712.g70f 2.1 MB/s | 290 kB 00:00 2026-03-20T11:20:39.137 INFO:teuthology.orchestra.run.vm07.stdout:(33/138): ceph-mgr-rook-20.2.0-712.g70f8415b.el 423 kB/s | 50 kB 00:00 2026-03-20T11:20:39.148 INFO:teuthology.orchestra.run.vm07.stdout:(34/138): ceph-prometheus-alerts-20.2.0-712.g70 136 kB/s | 17 kB 00:00 2026-03-20T11:20:39.255 INFO:teuthology.orchestra.run.vm05.stdout:(32/138): ceph-mgr-rook-20.2.0-712.g70f8415b.el 408 kB/s | 50 kB 00:00 2026-03-20T11:20:39.259 INFO:teuthology.orchestra.run.vm07.stdout:(35/138): ceph-volume-20.2.0-712.g70f8415b.el9. 2.4 MB/s | 298 kB 00:00 2026-03-20T11:20:39.297 INFO:teuthology.orchestra.run.vm07.stdout:(36/138): cephadm-20.2.0-712.g70f8415b.el9.noar 6.7 MB/s | 1.0 MB 00:00 2026-03-20T11:20:39.398 INFO:teuthology.orchestra.run.vm05.stdout:(33/138): ceph-prometheus-alerts-20.2.0-712.g70 122 kB/s | 17 kB 00:00 2026-03-20T11:20:39.527 INFO:teuthology.orchestra.run.vm05.stdout:(34/138): ceph-volume-20.2.0-712.g70f8415b.el9. 2.3 MB/s | 298 kB 00:00 2026-03-20T11:20:39.543 INFO:teuthology.orchestra.run.vm07.stdout:(37/138): bzip2-1.0.8-11.el9.x86_64.rpm 193 kB/s | 55 kB 00:00 2026-03-20T11:20:39.641 INFO:teuthology.orchestra.run.vm07.stdout:(38/138): cryptsetup-2.8.1-3.el9.x86_64.rpm 1.0 MB/s | 351 kB 00:00 2026-03-20T11:20:39.689 INFO:teuthology.orchestra.run.vm07.stdout:(39/138): fuse-2.9.9-17.el9.x86_64.rpm 549 kB/s | 80 kB 00:00 2026-03-20T11:20:39.691 INFO:teuthology.orchestra.run.vm07.stdout:(40/138): ledmon-libs-1.1.0-3.el9.x86_64.rpm 811 kB/s | 40 kB 00:00 2026-03-20T11:20:39.753 INFO:teuthology.orchestra.run.vm02.stdout:(30/138): ceph-mgr-diskprediction-local-20.2.0- 3.4 MB/s | 7.4 MB 00:02 2026-03-20T11:20:39.791 INFO:teuthology.orchestra.run.vm05.stdout:(35/138): cephadm-20.2.0-712.g70f8415b.el9.noar 3.8 MB/s | 1.0 MB 00:00 2026-03-20T11:20:39.817 INFO:teuthology.orchestra.run.vm07.stdout:(41/138): libconfig-1.7.2-9.el9.x86_64.rpm 565 kB/s | 72 kB 00:00 2026-03-20T11:20:39.878 INFO:teuthology.orchestra.run.vm02.stdout:(31/138): ceph-mgr-modules-core-20.2.0-712.g70f 2.3 MB/s | 290 kB 00:00 2026-03-20T11:20:39.938 INFO:teuthology.orchestra.run.vm05.stdout:(36/138): bzip2-1.0.8-11.el9.x86_64.rpm 373 kB/s | 55 kB 00:00 2026-03-20T11:20:39.997 INFO:teuthology.orchestra.run.vm02.stdout:(32/138): ceph-mgr-rook-20.2.0-712.g70f8415b.el 422 kB/s | 50 kB 00:00 2026-03-20T11:20:40.026 INFO:teuthology.orchestra.run.vm05.stdout:(37/138): cryptsetup-2.8.1-3.el9.x86_64.rpm 3.9 MB/s | 351 kB 00:00 2026-03-20T11:20:40.056 INFO:teuthology.orchestra.run.vm05.stdout:(38/138): fuse-2.9.9-17.el9.x86_64.rpm 2.6 MB/s | 80 kB 00:00 2026-03-20T11:20:40.072 INFO:teuthology.orchestra.run.vm07.stdout:(42/138): libgfortran-11.5.0-14.el9.x86_64.rpm 2.0 MB/s | 794 kB 00:00 2026-03-20T11:20:40.086 INFO:teuthology.orchestra.run.vm05.stdout:(39/138): ledmon-libs-1.1.0-3.el9.x86_64.rpm 1.3 MB/s | 40 kB 00:00 2026-03-20T11:20:40.116 INFO:teuthology.orchestra.run.vm05.stdout:(40/138): libconfig-1.7.2-9.el9.x86_64.rpm 2.4 MB/s | 72 kB 00:00 2026-03-20T11:20:40.116 INFO:teuthology.orchestra.run.vm02.stdout:(33/138): ceph-prometheus-alerts-20.2.0-712.g70 145 kB/s | 17 kB 00:00 2026-03-20T11:20:40.126 INFO:teuthology.orchestra.run.vm07.stdout:(43/138): libquadmath-11.5.0-14.el9.x86_64.rpm 596 kB/s | 184 kB 00:00 2026-03-20T11:20:40.128 INFO:teuthology.orchestra.run.vm07.stdout:(44/138): mailcap-2.1.49-5.el9.noarch.rpm 605 kB/s | 33 kB 00:00 2026-03-20T11:20:40.141 INFO:teuthology.orchestra.run.vm05.stdout:(41/138): ceph-mgr-dashboard-20.2.0-712.g70f841 4.1 MB/s | 11 MB 00:02 2026-03-20T11:20:40.158 INFO:teuthology.orchestra.run.vm02.stdout:(34/138): ceph-mgr-dashboard-20.2.0-712.g70f841 3.7 MB/s | 11 MB 00:02 2026-03-20T11:20:40.179 INFO:teuthology.orchestra.run.vm05.stdout:(42/138): libgfortran-11.5.0-14.el9.x86_64.rpm 12 MB/s | 794 kB 00:00 2026-03-20T11:20:40.180 INFO:teuthology.orchestra.run.vm05.stdout:(43/138): libquadmath-11.5.0-14.el9.x86_64.rpm 4.7 MB/s | 184 kB 00:00 2026-03-20T11:20:40.195 INFO:teuthology.orchestra.run.vm07.stdout:(45/138): pciutils-3.7.0-7.el9.x86_64.rpm 1.3 MB/s | 93 kB 00:00 2026-03-20T11:20:40.208 INFO:teuthology.orchestra.run.vm05.stdout:(44/138): mailcap-2.1.49-5.el9.noarch.rpm 1.1 MB/s | 33 kB 00:00 2026-03-20T11:20:40.210 INFO:teuthology.orchestra.run.vm05.stdout:(45/138): pciutils-3.7.0-7.el9.x86_64.rpm 3.1 MB/s | 93 kB 00:00 2026-03-20T11:20:40.236 INFO:teuthology.orchestra.run.vm02.stdout:(35/138): ceph-volume-20.2.0-712.g70f8415b.el9. 2.4 MB/s | 298 kB 00:00 2026-03-20T11:20:40.389 INFO:teuthology.orchestra.run.vm02.stdout:(36/138): cephadm-20.2.0-712.g70f8415b.el9.noar 4.3 MB/s | 1.0 MB 00:00 2026-03-20T11:20:40.504 INFO:teuthology.orchestra.run.vm05.stdout:(46/138): ceph-test-20.2.0-712.g70f8415b.el9.x8 12 MB/s | 84 MB 00:06 2026-03-20T11:20:40.555 INFO:teuthology.orchestra.run.vm02.stdout:(37/138): bzip2-1.0.8-11.el9.x86_64.rpm 172 kB/s | 55 kB 00:00 2026-03-20T11:20:40.666 INFO:teuthology.orchestra.run.vm08.stdout:(8/138): ceph-common-20.2.0-712.g70f8415b.el9.x 3.8 MB/s | 24 MB 00:06 2026-03-20T11:20:40.713 INFO:teuthology.orchestra.run.vm02.stdout:(38/138): fuse-2.9.9-17.el9.x86_64.rpm 506 kB/s | 80 kB 00:00 2026-03-20T11:20:40.719 INFO:teuthology.orchestra.run.vm05.stdout:(47/138): python3-cffi-1.14.5-5.el9.x86_64.rpm 496 kB/s | 253 kB 00:00 2026-03-20T11:20:40.742 INFO:teuthology.orchestra.run.vm07.stdout:(46/138): python3-cffi-1.14.5-5.el9.x86_64.rpm 412 kB/s | 253 kB 00:00 2026-03-20T11:20:40.787 INFO:teuthology.orchestra.run.vm05.stdout:(48/138): python3-ply-3.11-14.el9.noarch.rpm 376 kB/s | 106 kB 00:00 2026-03-20T11:20:40.788 INFO:teuthology.orchestra.run.vm08.stdout:(9/138): ceph-selinux-20.2.0-712.g70f8415b.el9. 205 kB/s | 25 kB 00:00 2026-03-20T11:20:40.810 INFO:teuthology.orchestra.run.vm02.stdout:(39/138): ledmon-libs-1.1.0-3.el9.x86_64.rpm 422 kB/s | 40 kB 00:00 2026-03-20T11:20:40.918 INFO:teuthology.orchestra.run.vm02.stdout:(40/138): libconfig-1.7.2-9.el9.x86_64.rpm 667 kB/s | 72 kB 00:00 2026-03-20T11:20:40.938 INFO:teuthology.orchestra.run.vm07.stdout:(47/138): python3-cryptography-36.0.1-5.el9.x86 1.7 MB/s | 1.2 MB 00:00 2026-03-20T11:20:40.939 INFO:teuthology.orchestra.run.vm07.stdout:(48/138): python3-ply-3.11-14.el9.noarch.rpm 542 kB/s | 106 kB 00:00 2026-03-20T11:20:40.986 INFO:teuthology.orchestra.run.vm02.stdout:(41/138): cryptsetup-2.8.1-3.el9.x86_64.rpm 588 kB/s | 351 kB 00:00 2026-03-20T11:20:41.050 INFO:teuthology.orchestra.run.vm07.stdout:(49/138): python3-pycparser-2.20-6.el9.noarch.r 1.2 MB/s | 135 kB 00:00 2026-03-20T11:20:41.103 INFO:teuthology.orchestra.run.vm02.stdout:(42/138): libquadmath-11.5.0-14.el9.x86_64.rpm 1.5 MB/s | 184 kB 00:00 2026-03-20T11:20:41.159 INFO:teuthology.orchestra.run.vm05.stdout:(49/138): python3-cryptography-36.0.1-5.el9.x86 1.3 MB/s | 1.2 MB 00:00 2026-03-20T11:20:41.184 INFO:teuthology.orchestra.run.vm05.stdout:(50/138): python3-pycparser-2.20-6.el9.noarch.r 291 kB/s | 135 kB 00:00 2026-03-20T11:20:41.188 INFO:teuthology.orchestra.run.vm02.stdout:(43/138): mailcap-2.1.49-5.el9.noarch.rpm 393 kB/s | 33 kB 00:00 2026-03-20T11:20:41.210 INFO:teuthology.orchestra.run.vm02.stdout:(44/138): libgfortran-11.5.0-14.el9.x86_64.rpm 2.7 MB/s | 794 kB 00:00 2026-03-20T11:20:41.210 INFO:teuthology.orchestra.run.vm07.stdout:(50/138): python3-pyparsing-2.4.7-9.el9.noarch. 556 kB/s | 150 kB 00:00 2026-03-20T11:20:41.210 INFO:teuthology.orchestra.run.vm05.stdout:(51/138): python3-pyparsing-2.4.7-9.el9.noarch. 356 kB/s | 150 kB 00:00 2026-03-20T11:20:41.263 INFO:teuthology.orchestra.run.vm07.stdout:(51/138): python3-requests-2.25.1-10.el9.noarch 593 kB/s | 126 kB 00:00 2026-03-20T11:20:41.264 INFO:teuthology.orchestra.run.vm05.stdout:(52/138): python3-requests-2.25.1-10.el9.noarch 1.2 MB/s | 126 kB 00:00 2026-03-20T11:20:41.300 INFO:teuthology.orchestra.run.vm02.stdout:(45/138): pciutils-3.7.0-7.el9.x86_64.rpm 832 kB/s | 93 kB 00:00 2026-03-20T11:20:41.344 INFO:teuthology.orchestra.run.vm02.stdout:(46/138): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.9 MB/s | 253 kB 00:00 2026-03-20T11:20:41.426 INFO:teuthology.orchestra.run.vm02.stdout:(47/138): python3-ply-3.11-14.el9.noarch.rpm 1.3 MB/s | 106 kB 00:00 2026-03-20T11:20:41.526 INFO:teuthology.orchestra.run.vm02.stdout:(48/138): python3-pycparser-2.20-6.el9.noarch.r 1.3 MB/s | 135 kB 00:00 2026-03-20T11:20:41.554 INFO:teuthology.orchestra.run.vm02.stdout:(49/138): python3-cryptography-36.0.1-5.el9.x86 4.9 MB/s | 1.2 MB 00:00 2026-03-20T11:20:41.661 INFO:teuthology.orchestra.run.vm02.stdout:(50/138): python3-pyparsing-2.4.7-9.el9.noarch. 1.1 MB/s | 150 kB 00:00 2026-03-20T11:20:41.695 INFO:teuthology.orchestra.run.vm02.stdout:(51/138): python3-requests-2.25.1-10.el9.noarch 896 kB/s | 126 kB 00:00 2026-03-20T11:20:41.785 INFO:teuthology.orchestra.run.vm05.stdout:(53/138): unzip-6.0-59.el9.x86_64.rpm 316 kB/s | 182 kB 00:00 2026-03-20T11:20:41.786 INFO:teuthology.orchestra.run.vm05.stdout:(54/138): python3-urllib3-1.26.5-7.el9.noarch.r 362 kB/s | 218 kB 00:00 2026-03-20T11:20:41.810 INFO:teuthology.orchestra.run.vm07.stdout:(52/138): python3-urllib3-1.26.5-7.el9.noarch.r 364 kB/s | 218 kB 00:00 2026-03-20T11:20:41.812 INFO:teuthology.orchestra.run.vm07.stdout:(53/138): unzip-6.0-59.el9.x86_64.rpm 332 kB/s | 182 kB 00:00 2026-03-20T11:20:41.821 INFO:teuthology.orchestra.run.vm02.stdout:(52/138): python3-urllib3-1.26.5-7.el9.noarch.r 1.3 MB/s | 218 kB 00:00 2026-03-20T11:20:41.847 INFO:teuthology.orchestra.run.vm02.stdout:(53/138): unzip-6.0-59.el9.x86_64.rpm 1.2 MB/s | 182 kB 00:00 2026-03-20T11:20:41.936 INFO:teuthology.orchestra.run.vm02.stdout:(54/138): zip-3.0-35.el9.x86_64.rpm 2.3 MB/s | 266 kB 00:00 2026-03-20T11:20:41.948 INFO:teuthology.orchestra.run.vm07.stdout:(54/138): boost-program-options-1.75.0-13.el9.x 768 kB/s | 104 kB 00:00 2026-03-20T11:20:41.948 INFO:teuthology.orchestra.run.vm02.stdout:(55/138): boost-program-options-1.75.0-13.el9.x 1.0 MB/s | 104 kB 00:00 2026-03-20T11:20:41.962 INFO:teuthology.orchestra.run.vm05.stdout:(55/138): flexiblas-3.0.4-9.el9.x86_64.rpm 169 kB/s | 30 kB 00:00 2026-03-20T11:20:42.011 INFO:teuthology.orchestra.run.vm07.stdout:(55/138): flexiblas-3.0.4-9.el9.x86_64.rpm 471 kB/s | 30 kB 00:00 2026-03-20T11:20:42.039 INFO:teuthology.orchestra.run.vm02.stdout:(56/138): flexiblas-3.0.4-9.el9.x86_64.rpm 287 kB/s | 30 kB 00:00 2026-03-20T11:20:42.046 INFO:teuthology.orchestra.run.vm05.stdout:(56/138): boost-program-options-1.75.0-13.el9.x 400 kB/s | 104 kB 00:00 2026-03-20T11:20:42.102 INFO:teuthology.orchestra.run.vm05.stdout:(57/138): flexiblas-openblas-openmp-3.0.4-9.el9 267 kB/s | 15 kB 00:00 2026-03-20T11:20:42.122 INFO:teuthology.orchestra.run.vm02.stdout:(57/138): flexiblas-openblas-openmp-3.0.4-9.el9 180 kB/s | 15 kB 00:00 2026-03-20T11:20:42.175 INFO:teuthology.orchestra.run.vm07.stdout:(56/138): flexiblas-netlib-3.0.4-9.el9.x86_64.r 18 MB/s | 3.0 MB 00:00 2026-03-20T11:20:42.203 INFO:teuthology.orchestra.run.vm07.stdout:(57/138): flexiblas-openblas-openmp-3.0.4-9.el9 529 kB/s | 15 kB 00:00 2026-03-20T11:20:42.224 INFO:teuthology.orchestra.run.vm05.stdout:(58/138): libnbd-1.20.3-4.el9.x86_64.rpm 1.3 MB/s | 164 kB 00:00 2026-03-20T11:20:42.230 INFO:teuthology.orchestra.run.vm07.stdout:(58/138): libnbd-1.20.3-4.el9.x86_64.rpm 6.2 MB/s | 164 kB 00:00 2026-03-20T11:20:42.234 INFO:teuthology.orchestra.run.vm02.stdout:(58/138): flexiblas-netlib-3.0.4-9.el9.x86_64.r 10 MB/s | 3.0 MB 00:00 2026-03-20T11:20:42.235 INFO:teuthology.orchestra.run.vm02.stdout:(59/138): libnbd-1.20.3-4.el9.x86_64.rpm 1.4 MB/s | 164 kB 00:00 2026-03-20T11:20:42.262 INFO:teuthology.orchestra.run.vm07.stdout:(59/138): libpmemobj-1.12.1-1.el9.x86_64.rpm 4.8 MB/s | 160 kB 00:00 2026-03-20T11:20:42.288 INFO:teuthology.orchestra.run.vm07.stdout:(60/138): librabbitmq-0.11.0-7.el9.x86_64.rpm 1.7 MB/s | 45 kB 00:00 2026-03-20T11:20:42.296 INFO:teuthology.orchestra.run.vm05.stdout:(59/138): libpmemobj-1.12.1-1.el9.x86_64.rpm 2.2 MB/s | 160 kB 00:00 2026-03-20T11:20:42.339 INFO:teuthology.orchestra.run.vm02.stdout:(60/138): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.5 MB/s | 160 kB 00:00 2026-03-20T11:20:42.357 INFO:teuthology.orchestra.run.vm07.stdout:(61/138): librdkafka-1.6.1-102.el9.x86_64.rpm 9.5 MB/s | 662 kB 00:00 2026-03-20T11:20:42.367 INFO:teuthology.orchestra.run.vm05.stdout:(60/138): librabbitmq-0.11.0-7.el9.x86_64.rpm 642 kB/s | 45 kB 00:00 2026-03-20T11:20:42.371 INFO:teuthology.orchestra.run.vm02.stdout:(61/138): librabbitmq-0.11.0-7.el9.x86_64.rpm 333 kB/s | 45 kB 00:00 2026-03-20T11:20:42.398 INFO:teuthology.orchestra.run.vm07.stdout:(62/138): libstoragemgmt-1.10.1-1.el9.x86_64.rp 5.9 MB/s | 246 kB 00:00 2026-03-20T11:20:42.422 INFO:teuthology.orchestra.run.vm05.stdout:(61/138): flexiblas-netlib-3.0.4-9.el9.x86_64.r 6.5 MB/s | 3.0 MB 00:00 2026-03-20T11:20:42.425 INFO:teuthology.orchestra.run.vm07.stdout:(63/138): libxslt-1.1.34-12.el9.x86_64.rpm 8.5 MB/s | 233 kB 00:00 2026-03-20T11:20:42.444 INFO:teuthology.orchestra.run.vm07.stdout:(64/138): zip-3.0-35.el9.x86_64.rpm 419 kB/s | 266 kB 00:00 2026-03-20T11:20:42.452 INFO:teuthology.orchestra.run.vm05.stdout:(62/138): zip-3.0-35.el9.x86_64.rpm 224 kB/s | 266 kB 00:01 2026-03-20T11:20:42.458 INFO:teuthology.orchestra.run.vm02.stdout:(62/138): librdkafka-1.6.1-102.el9.x86_64.rpm 5.5 MB/s | 662 kB 00:00 2026-03-20T11:20:42.459 INFO:teuthology.orchestra.run.vm07.stdout:(65/138): lttng-ust-2.12.0-6.el9.x86_64.rpm 8.3 MB/s | 292 kB 00:00 2026-03-20T11:20:42.467 INFO:teuthology.orchestra.run.vm02.stdout:(63/138): libstoragemgmt-1.10.1-1.el9.x86_64.rp 2.5 MB/s | 246 kB 00:00 2026-03-20T11:20:42.471 INFO:teuthology.orchestra.run.vm05.stdout:(63/138): librdkafka-1.6.1-102.el9.x86_64.rpm 6.2 MB/s | 662 kB 00:00 2026-03-20T11:20:42.498 INFO:teuthology.orchestra.run.vm07.stdout:(66/138): openblas-0.3.29-1.el9.x86_64.rpm 1.2 MB/s | 42 kB 00:00 2026-03-20T11:20:42.506 INFO:teuthology.orchestra.run.vm05.stdout:(64/138): libstoragemgmt-1.10.1-1.el9.x86_64.rp 2.9 MB/s | 246 kB 00:00 2026-03-20T11:20:42.562 INFO:teuthology.orchestra.run.vm05.stdout:(65/138): lttng-ust-2.12.0-6.el9.x86_64.rpm 3.2 MB/s | 292 kB 00:00 2026-03-20T11:20:42.568 INFO:teuthology.orchestra.run.vm02.stdout:(64/138): libxslt-1.1.34-12.el9.x86_64.rpm 2.1 MB/s | 233 kB 00:00 2026-03-20T11:20:42.592 INFO:teuthology.orchestra.run.vm07.stdout:(67/138): lua-5.4.4-4.el9.x86_64.rpm 1.2 MB/s | 188 kB 00:00 2026-03-20T11:20:42.602 INFO:teuthology.orchestra.run.vm02.stdout:(65/138): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.1 MB/s | 292 kB 00:00 2026-03-20T11:20:42.629 INFO:teuthology.orchestra.run.vm07.stdout:(68/138): perl-Benchmark-1.23-483.el9.noarch.rp 735 kB/s | 26 kB 00:00 2026-03-20T11:20:42.631 INFO:teuthology.orchestra.run.vm05.stdout:(66/138): openblas-0.3.29-1.el9.x86_64.rpm 614 kB/s | 42 kB 00:00 2026-03-20T11:20:42.642 INFO:teuthology.orchestra.run.vm05.stdout:(67/138): lua-5.4.4-4.el9.x86_64.rpm 1.4 MB/s | 188 kB 00:00 2026-03-20T11:20:42.670 INFO:teuthology.orchestra.run.vm02.stdout:(66/138): lua-5.4.4-4.el9.x86_64.rpm 1.8 MB/s | 188 kB 00:00 2026-03-20T11:20:42.673 INFO:teuthology.orchestra.run.vm07.stdout:(69/138): perl-Test-Harness-3.42-461.el9.noarch 6.6 MB/s | 295 kB 00:00 2026-03-20T11:20:42.681 INFO:teuthology.orchestra.run.vm02.stdout:(67/138): openblas-0.3.29-1.el9.x86_64.rpm 541 kB/s | 42 kB 00:00 2026-03-20T11:20:42.703 INFO:teuthology.orchestra.run.vm05.stdout:(68/138): perl-Benchmark-1.23-483.el9.noarch.rp 434 kB/s | 26 kB 00:00 2026-03-20T11:20:42.740 INFO:teuthology.orchestra.run.vm07.stdout:(70/138): openblas-openmp-0.3.29-1.el9.x86_64.r 22 MB/s | 5.3 MB 00:00 2026-03-20T11:20:42.746 INFO:teuthology.orchestra.run.vm07.stdout:(71/138): protobuf-3.14.0-17.el9.x86_64.rpm 14 MB/s | 1.0 MB 00:00 2026-03-20T11:20:42.762 INFO:teuthology.orchestra.run.vm05.stdout:(69/138): libxslt-1.1.34-12.el9.x86_64.rpm 754 kB/s | 233 kB 00:00 2026-03-20T11:20:42.764 INFO:teuthology.orchestra.run.vm02.stdout:(68/138): perl-Benchmark-1.23-483.el9.noarch.rp 320 kB/s | 26 kB 00:00 2026-03-20T11:20:42.797 INFO:teuthology.orchestra.run.vm05.stdout:(70/138): perl-Test-Harness-3.42-461.el9.noarch 3.1 MB/s | 295 kB 00:00 2026-03-20T11:20:42.868 INFO:teuthology.orchestra.run.vm02.stdout:(69/138): perl-Test-Harness-3.42-461.el9.noarch 2.8 MB/s | 295 kB 00:00 2026-03-20T11:20:42.982 INFO:teuthology.orchestra.run.vm05.stdout:(71/138): protobuf-3.14.0-17.el9.x86_64.rpm 4.6 MB/s | 1.0 MB 00:00 2026-03-20T11:20:42.986 INFO:teuthology.orchestra.run.vm08.stdout:(10/138): ceph-radosgw-20.2.0-712.g70f8415b.el9 5.2 MB/s | 24 MB 00:04 2026-03-20T11:20:42.986 INFO:teuthology.orchestra.run.vm07.stdout:(72/138): ceph-test-20.2.0-712.g70f8415b.el9.x8 9.2 MB/s | 84 MB 00:09 2026-03-20T11:20:42.989 INFO:teuthology.orchestra.run.vm07.stdout:(73/138): python3-devel-3.9.25-3.el9.x86_64.rpm 1.0 MB/s | 244 kB 00:00 2026-03-20T11:20:43.011 INFO:teuthology.orchestra.run.vm02.stdout:(70/138): openblas-openmp-0.3.29-1.el9.x86_64.r 16 MB/s | 5.3 MB 00:00 2026-03-20T11:20:43.017 INFO:teuthology.orchestra.run.vm07.stdout:(74/138): python3-jmespath-1.0.1-1.el9.noarch.r 1.7 MB/s | 48 kB 00:00 2026-03-20T11:20:43.050 INFO:teuthology.orchestra.run.vm07.stdout:(75/138): python3-libstoragemgmt-1.10.1-1.el9.x 5.3 MB/s | 177 kB 00:00 2026-03-20T11:20:43.082 INFO:teuthology.orchestra.run.vm07.stdout:(76/138): python3-markupsafe-1.1.1-12.el9.x86_6 1.1 MB/s | 35 kB 00:00 2026-03-20T11:20:43.092 INFO:teuthology.orchestra.run.vm05.stdout:(72/138): python3-devel-3.9.25-3.el9.x86_64.rpm 2.2 MB/s | 244 kB 00:00 2026-03-20T11:20:43.095 INFO:teuthology.orchestra.run.vm08.stdout:(11/138): libcephfs-devel-20.2.0-712.g70f8415b. 316 kB/s | 34 kB 00:00 2026-03-20T11:20:43.117 INFO:teuthology.orchestra.run.vm07.stdout:(77/138): python3-jinja2-2.11.3-8.el9.noarch.rp 1.9 MB/s | 249 kB 00:00 2026-03-20T11:20:43.184 INFO:teuthology.orchestra.run.vm05.stdout:(73/138): python3-jinja2-2.11.3-8.el9.noarch.rp 2.7 MB/s | 249 kB 00:00 2026-03-20T11:20:43.203 INFO:teuthology.orchestra.run.vm08.stdout:(12/138): libcephfs-proxy2-20.2.0-712.g70f8415b 223 kB/s | 24 kB 00:00 2026-03-20T11:20:43.251 INFO:teuthology.orchestra.run.vm05.stdout:(74/138): python3-jmespath-1.0.1-1.el9.noarch.r 708 kB/s | 48 kB 00:00 2026-03-20T11:20:43.264 INFO:teuthology.orchestra.run.vm07.stdout:(78/138): python3-numpy-f2py-1.23.5-2.el9.x86_6 2.9 MB/s | 442 kB 00:00 2026-03-20T11:20:43.299 INFO:teuthology.orchestra.run.vm07.stdout:(79/138): python3-packaging-20.9-5.el9.noarch.r 2.2 MB/s | 77 kB 00:00 2026-03-20T11:20:43.305 INFO:teuthology.orchestra.run.vm05.stdout:(75/138): openblas-openmp-0.3.29-1.el9.x86_64.r 7.8 MB/s | 5.3 MB 00:00 2026-03-20T11:20:43.338 INFO:teuthology.orchestra.run.vm07.stdout:(80/138): python3-protobuf-3.14.0-17.el9.noarch 6.6 MB/s | 267 kB 00:00 2026-03-20T11:20:43.347 INFO:teuthology.orchestra.run.vm05.stdout:(76/138): python3-libstoragemgmt-1.10.1-1.el9.x 1.8 MB/s | 177 kB 00:00 2026-03-20T11:20:43.365 INFO:teuthology.orchestra.run.vm05.stdout:(77/138): python3-markupsafe-1.1.1-12.el9.x86_6 586 kB/s | 35 kB 00:00 2026-03-20T11:20:43.395 INFO:teuthology.orchestra.run.vm07.stdout:(81/138): python3-pyasn1-0.4.8-7.el9.noarch.rpm 2.7 MB/s | 157 kB 00:00 2026-03-20T11:20:43.427 INFO:teuthology.orchestra.run.vm07.stdout:(82/138): python3-numpy-1.23.5-2.el9.x86_64.rpm 18 MB/s | 6.1 MB 00:00 2026-03-20T11:20:43.429 INFO:teuthology.orchestra.run.vm08.stdout:(13/138): libcephfs2-20.2.0-712.g70f8415b.el9.x 3.8 MB/s | 866 kB 00:00 2026-03-20T11:20:43.456 INFO:teuthology.orchestra.run.vm07.stdout:(83/138): python3-requests-oauthlib-1.3.0-12.el 1.8 MB/s | 54 kB 00:00 2026-03-20T11:20:43.465 INFO:teuthology.orchestra.run.vm05.stdout:(78/138): python3-numpy-f2py-1.23.5-2.el9.x86_6 4.3 MB/s | 442 kB 00:00 2026-03-20T11:20:43.513 INFO:teuthology.orchestra.run.vm07.stdout:(84/138): python3-babel-2.9.1-2.el9.noarch.rpm 7.7 MB/s | 6.0 MB 00:00 2026-03-20T11:20:43.516 INFO:teuthology.orchestra.run.vm05.stdout:(79/138): python3-packaging-20.9-5.el9.noarch.r 1.5 MB/s | 77 kB 00:00 2026-03-20T11:20:43.538 INFO:teuthology.orchestra.run.vm07.stdout:(85/138): python3-pyasn1-modules-0.4.8-7.el9.no 1.9 MB/s | 277 kB 00:00 2026-03-20T11:20:43.539 INFO:teuthology.orchestra.run.vm07.stdout:(86/138): python3-toml-0.10.2-6.el9.noarch.rpm 1.6 MB/s | 42 kB 00:00 2026-03-20T11:20:43.540 INFO:teuthology.orchestra.run.vm08.stdout:(14/138): libcephsqlite-20.2.0-712.g70f8415b.el 1.5 MB/s | 164 kB 00:00 2026-03-20T11:20:43.566 INFO:teuthology.orchestra.run.vm07.stdout:(87/138): qatlib-service-25.08.0-2.el9.x86_64.r 1.4 MB/s | 37 kB 00:00 2026-03-20T11:20:43.593 INFO:teuthology.orchestra.run.vm05.stdout:(80/138): python3-protobuf-3.14.0-17.el9.noarch 3.4 MB/s | 267 kB 00:00 2026-03-20T11:20:43.595 INFO:teuthology.orchestra.run.vm07.stdout:(88/138): qatlib-25.08.0-2.el9.x86_64.rpm 4.1 MB/s | 240 kB 00:00 2026-03-20T11:20:43.628 INFO:teuthology.orchestra.run.vm07.stdout:(89/138): qatzip-libs-1.3.1-1.el9.x86_64.rpm 1.1 MB/s | 66 kB 00:00 2026-03-20T11:20:43.630 INFO:teuthology.orchestra.run.vm02.stdout:(71/138): protobuf-3.14.0-17.el9.x86_64.rpm 1.3 MB/s | 1.0 MB 00:00 2026-03-20T11:20:43.645 INFO:teuthology.orchestra.run.vm05.stdout:(81/138): python3-pyasn1-0.4.8-7.el9.noarch.rpm 3.0 MB/s | 157 kB 00:00 2026-03-20T11:20:43.651 INFO:teuthology.orchestra.run.vm08.stdout:(15/138): librados-devel-20.2.0-712.g70f8415b.e 1.1 MB/s | 126 kB 00:00 2026-03-20T11:20:43.653 INFO:teuthology.orchestra.run.vm07.stdout:(90/138): socat-1.7.4.1-8.el9.x86_64.rpm 5.2 MB/s | 303 kB 00:00 2026-03-20T11:20:43.682 INFO:teuthology.orchestra.run.vm07.stdout:(91/138): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.2 MB/s | 64 kB 00:00 2026-03-20T11:20:43.698 INFO:teuthology.orchestra.run.vm05.stdout:(82/138): python3-pyasn1-modules-0.4.8-7.el9.no 5.1 MB/s | 277 kB 00:00 2026-03-20T11:20:43.763 INFO:teuthology.orchestra.run.vm08.stdout:(16/138): libradosstriper1-20.2.0-712.g70f8415b 2.2 MB/s | 250 kB 00:00 2026-03-20T11:20:43.769 INFO:teuthology.orchestra.run.vm05.stdout:(83/138): python3-requests-oauthlib-1.3.0-12.el 765 kB/s | 54 kB 00:00 2026-03-20T11:20:43.789 INFO:teuthology.orchestra.run.vm02.stdout:(72/138): python3-babel-2.9.1-2.el9.noarch.rpm 7.7 MB/s | 6.0 MB 00:00 2026-03-20T11:20:43.807 INFO:teuthology.orchestra.run.vm07.stdout:(92/138): lua-devel-5.4.4-4.el9.x86_64.rpm 145 kB/s | 22 kB 00:00 2026-03-20T11:20:43.814 INFO:teuthology.orchestra.run.vm02.stdout:(73/138): python3-devel-3.9.25-3.el9.x86_64.rpm 1.3 MB/s | 244 kB 00:00 2026-03-20T11:20:43.914 INFO:teuthology.orchestra.run.vm07.stdout:(93/138): protobuf-compiler-3.14.0-17.el9.x86_6 3.6 MB/s | 862 kB 00:00 2026-03-20T11:20:43.942 INFO:teuthology.orchestra.run.vm02.stdout:(74/138): python3-jmespath-1.0.1-1.el9.noarch.r 374 kB/s | 48 kB 00:00 2026-03-20T11:20:43.942 INFO:teuthology.orchestra.run.vm05.stdout:(84/138): python3-numpy-1.23.5-2.el9.x86_64.rpm 10 MB/s | 6.1 MB 00:00 2026-03-20T11:20:44.040 INFO:teuthology.orchestra.run.vm05.stdout:(85/138): python3-toml-0.10.2-6.el9.noarch.rpm 430 kB/s | 42 kB 00:00 2026-03-20T11:20:44.042 INFO:teuthology.orchestra.run.vm02.stdout:(75/138): python3-jinja2-2.11.3-8.el9.noarch.rp 984 kB/s | 249 kB 00:00 2026-03-20T11:20:44.068 INFO:teuthology.orchestra.run.vm02.stdout:(76/138): python3-libstoragemgmt-1.10.1-1.el9.x 1.4 MB/s | 177 kB 00:00 2026-03-20T11:20:44.091 INFO:teuthology.orchestra.run.vm07.stdout:(94/138): abseil-cpp-20211102.0-4.el9.x86_64.rp 1.9 MB/s | 551 kB 00:00 2026-03-20T11:20:44.121 INFO:teuthology.orchestra.run.vm05.stdout:(86/138): qatlib-25.08.0-2.el9.x86_64.rpm 2.9 MB/s | 240 kB 00:00 2026-03-20T11:20:44.142 INFO:teuthology.orchestra.run.vm07.stdout:(95/138): grpc-data-1.46.7-10.el9.noarch.rpm 379 kB/s | 19 kB 00:00 2026-03-20T11:20:44.150 INFO:teuthology.orchestra.run.vm07.stdout:(96/138): gperftools-libs-2.9.1-3.el9.x86_64.rp 1.3 MB/s | 308 kB 00:00 2026-03-20T11:20:44.190 INFO:teuthology.orchestra.run.vm05.stdout:(87/138): qatlib-service-25.08.0-2.el9.x86_64.r 540 kB/s | 37 kB 00:00 2026-03-20T11:20:44.230 INFO:teuthology.orchestra.run.vm07.stdout:(97/138): libarrow-doc-9.0.0-15.el9.noarch.rpm 311 kB/s | 25 kB 00:00 2026-03-20T11:20:44.262 INFO:teuthology.orchestra.run.vm05.stdout:(88/138): qatzip-libs-1.3.1-1.el9.x86_64.rpm 925 kB/s | 66 kB 00:00 2026-03-20T11:20:44.277 INFO:teuthology.orchestra.run.vm07.stdout:(98/138): liboath-2.6.12-1.el9.x86_64.rpm 1.0 MB/s | 49 kB 00:00 2026-03-20T11:20:44.327 INFO:teuthology.orchestra.run.vm07.stdout:(99/138): libunwind-1.6.2-1.el9.x86_64.rpm 1.3 MB/s | 67 kB 00:00 2026-03-20T11:20:44.329 INFO:teuthology.orchestra.run.vm02.stdout:(77/138): python3-markupsafe-1.1.1-12.el9.x86_6 121 kB/s | 35 kB 00:00 2026-03-20T11:20:44.347 INFO:teuthology.orchestra.run.vm07.stdout:(100/138): libarrow-9.0.0-15.el9.x86_64.rpm 22 MB/s | 4.4 MB 00:00 2026-03-20T11:20:44.351 INFO:teuthology.orchestra.run.vm05.stdout:(89/138): socat-1.7.4.1-8.el9.x86_64.rpm 3.3 MB/s | 303 kB 00:00 2026-03-20T11:20:44.378 INFO:teuthology.orchestra.run.vm07.stdout:(101/138): luarocks-3.9.2-5.el9.noarch.rpm 2.9 MB/s | 151 kB 00:00 2026-03-20T11:20:44.422 INFO:teuthology.orchestra.run.vm07.stdout:(102/138): parquet-libs-9.0.0-15.el9.x86_64.rpm 11 MB/s | 838 kB 00:00 2026-03-20T11:20:44.439 INFO:teuthology.orchestra.run.vm05.stdout:(90/138): python3-babel-2.9.1-2.el9.noarch.rpm 3.6 MB/s | 6.0 MB 00:01 2026-03-20T11:20:44.440 INFO:teuthology.orchestra.run.vm05.stdout:(91/138): xmlstarlet-1.6.1-20.el9.x86_64.rpm 719 kB/s | 64 kB 00:00 2026-03-20T11:20:44.472 INFO:teuthology.orchestra.run.vm07.stdout:(103/138): python3-asyncssh-2.13.2-5.el9.noarch 5.8 MB/s | 548 kB 00:00 2026-03-20T11:20:44.472 INFO:teuthology.orchestra.run.vm07.stdout:(104/138): python3-autocommand-2.2.2-8.el9.noar 581 kB/s | 29 kB 00:00 2026-03-20T11:20:44.522 INFO:teuthology.orchestra.run.vm07.stdout:(105/138): python3-backports-tarfile-1.2.0-1.el 1.2 MB/s | 60 kB 00:00 2026-03-20T11:20:44.524 INFO:teuthology.orchestra.run.vm07.stdout:(106/138): python3-bcrypt-3.2.2-1.el9.x86_64.rp 851 kB/s | 43 kB 00:00 2026-03-20T11:20:44.543 INFO:teuthology.orchestra.run.vm02.stdout:(78/138): python3-numpy-f2py-1.23.5-2.el9.x86_6 2.0 MB/s | 442 kB 00:00 2026-03-20T11:20:44.574 INFO:teuthology.orchestra.run.vm07.stdout:(107/138): python3-cachetools-4.2.4-1.el9.noarc 630 kB/s | 32 kB 00:00 2026-03-20T11:20:44.590 INFO:teuthology.orchestra.run.vm07.stdout:(108/138): python3-certifi-2023.05.07-4.el9.noa 215 kB/s | 14 kB 00:00 2026-03-20T11:20:44.617 INFO:teuthology.orchestra.run.vm07.stdout:(109/138): python3-cheroot-10.0.1-4.el9.noarch. 3.9 MB/s | 173 kB 00:00 2026-03-20T11:20:44.665 INFO:teuthology.orchestra.run.vm02.stdout:(79/138): python3-numpy-1.23.5-2.el9.x86_64.rpm 10 MB/s | 6.1 MB 00:00 2026-03-20T11:20:44.678 INFO:teuthology.orchestra.run.vm02.stdout:(80/138): python3-packaging-20.9-5.el9.noarch.r 571 kB/s | 77 kB 00:00 2026-03-20T11:20:44.678 INFO:teuthology.orchestra.run.vm07.stdout:(110/138): python3-scipy-1.9.3-2.el9.x86_64.rpm 16 MB/s | 19 MB 00:01 2026-03-20T11:20:44.682 INFO:teuthology.orchestra.run.vm07.stdout:(111/138): python3-google-auth-2.45.0-1.el9.noa 3.8 MB/s | 254 kB 00:00 2026-03-20T11:20:44.685 INFO:teuthology.orchestra.run.vm07.stdout:(112/138): python3-cherrypy-18.6.1-2.el9.noarch 3.7 MB/s | 358 kB 00:00 2026-03-20T11:20:44.732 INFO:teuthology.orchestra.run.vm07.stdout:(113/138): python3-grpcio-tools-1.46.7-10.el9.x 2.8 MB/s | 144 kB 00:00 2026-03-20T11:20:44.749 INFO:teuthology.orchestra.run.vm07.stdout:(114/138): python3-jaraco-8.2.1-3.el9.noarch.rp 168 kB/s | 11 kB 00:00 2026-03-20T11:20:44.773 INFO:teuthology.orchestra.run.vm02.stdout:(81/138): python3-protobuf-3.14.0-17.el9.noarch 2.4 MB/s | 267 kB 00:00 2026-03-20T11:20:44.774 INFO:teuthology.orchestra.run.vm07.stdout:(115/138): python3-jaraco-classes-3.2.1-5.el9.n 428 kB/s | 18 kB 00:00 2026-03-20T11:20:44.780 INFO:teuthology.orchestra.run.vm07.stdout:(116/138): python3-jaraco-collections-3.0.0-8.e 748 kB/s | 23 kB 00:00 2026-03-20T11:20:44.811 INFO:teuthology.orchestra.run.vm02.stdout:(82/138): python3-pyasn1-0.4.8-7.el9.noarch.rpm 1.2 MB/s | 157 kB 00:00 2026-03-20T11:20:44.820 INFO:teuthology.orchestra.run.vm07.stdout:(117/138): python3-jaraco-context-6.0.1-3.el9.n 432 kB/s | 20 kB 00:00 2026-03-20T11:20:44.821 INFO:teuthology.orchestra.run.vm07.stdout:(118/138): python3-jaraco-functools-3.5.0-2.el9 477 kB/s | 19 kB 00:00 2026-03-20T11:20:44.863 INFO:teuthology.orchestra.run.vm07.stdout:(119/138): python3-jaraco-text-4.0.0-2.el9.noar 612 kB/s | 26 kB 00:00 2026-03-20T11:20:44.893 INFO:teuthology.orchestra.run.vm02.stdout:(83/138): python3-pyasn1-modules-0.4.8-7.el9.no 2.2 MB/s | 277 kB 00:00 2026-03-20T11:20:44.918 INFO:teuthology.orchestra.run.vm05.stdout:(92/138): lua-devel-5.4.4-4.el9.x86_64.rpm 47 kB/s | 22 kB 00:00 2026-03-20T11:20:44.920 INFO:teuthology.orchestra.run.vm07.stdout:(120/138): python3-kubernetes-26.1.0-3.el9.noar 10 MB/s | 1.0 MB 00:00 2026-03-20T11:20:44.933 INFO:teuthology.orchestra.run.vm07.stdout:(121/138): python3-more-itertools-8.12.0-2.el9. 1.1 MB/s | 79 kB 00:00 2026-03-20T11:20:44.969 INFO:teuthology.orchestra.run.vm07.stdout:(122/138): python3-grpcio-1.46.7-10.el9.x86_64. 7.0 MB/s | 2.0 MB 00:00 2026-03-20T11:20:44.972 INFO:teuthology.orchestra.run.vm07.stdout:(123/138): python3-natsort-7.1.1-5.el9.noarch.r 1.1 MB/s | 58 kB 00:00 2026-03-20T11:20:44.988 INFO:teuthology.orchestra.run.vm07.stdout:(124/138): python3-portend-3.1.0-2.el9.noarch.r 303 kB/s | 16 kB 00:00 2026-03-20T11:20:45.019 INFO:teuthology.orchestra.run.vm07.stdout:(125/138): python3-pyOpenSSL-21.0.0-1.el9.noarc 1.8 MB/s | 90 kB 00:00 2026-03-20T11:20:45.020 INFO:teuthology.orchestra.run.vm07.stdout:(126/138): python3-repoze-lru-0.7-16.el9.noarch 641 kB/s | 31 kB 00:00 2026-03-20T11:20:45.044 INFO:teuthology.orchestra.run.vm07.stdout:(127/138): python3-routes-2.5.1-5.el9.noarch.rp 3.3 MB/s | 188 kB 00:00 2026-03-20T11:20:45.051 INFO:teuthology.orchestra.run.vm07.stdout:(128/138): python3-rsa-4.9-2.el9.noarch.rpm 1.8 MB/s | 59 kB 00:00 2026-03-20T11:20:45.053 INFO:teuthology.orchestra.run.vm07.stdout:(129/138): python3-tempora-5.0.0-2.el9.noarch.r 1.1 MB/s | 36 kB 00:00 2026-03-20T11:20:45.098 INFO:teuthology.orchestra.run.vm07.stdout:(130/138): python3-typing-extensions-4.15.0-1.e 1.6 MB/s | 86 kB 00:00 2026-03-20T11:20:45.101 INFO:teuthology.orchestra.run.vm07.stdout:(131/138): python3-websocket-client-1.2.3-2.el9 1.8 MB/s | 90 kB 00:00 2026-03-20T11:20:45.107 INFO:teuthology.orchestra.run.vm05.stdout:(93/138): abseil-cpp-20211102.0-4.el9.x86_64.rp 2.9 MB/s | 551 kB 00:00 2026-03-20T11:20:45.118 INFO:teuthology.orchestra.run.vm02.stdout:(84/138): python3-requests-oauthlib-1.3.0-12.el 175 kB/s | 54 kB 00:00 2026-03-20T11:20:45.120 INFO:teuthology.orchestra.run.vm07.stdout:(132/138): python3-xmltodict-0.12.0-15.el9.noar 330 kB/s | 22 kB 00:00 2026-03-20T11:20:45.138 INFO:teuthology.orchestra.run.vm05.stdout:(94/138): gperftools-libs-2.9.1-3.el9.x86_64.rp 9.7 MB/s | 308 kB 00:00 2026-03-20T11:20:45.142 INFO:teuthology.orchestra.run.vm07.stdout:(133/138): python3-zc-lockfile-2.0-10.el9.noarc 456 kB/s | 20 kB 00:00 2026-03-20T11:20:45.155 INFO:teuthology.orchestra.run.vm07.stdout:(134/138): re2-20211101-20.el9.x86_64.rpm 3.6 MB/s | 191 kB 00:00 2026-03-20T11:20:45.166 INFO:teuthology.orchestra.run.vm05.stdout:(95/138): grpc-data-1.46.7-10.el9.noarch.rpm 705 kB/s | 19 kB 00:00 2026-03-20T11:20:45.180 INFO:teuthology.orchestra.run.vm07.stdout:(135/138): s3cmd-2.4.0-1.el9.noarch.rpm 3.4 MB/s | 206 kB 00:00 2026-03-20T11:20:45.233 INFO:teuthology.orchestra.run.vm02.stdout:(85/138): python3-toml-0.10.2-6.el9.noarch.rpm 364 kB/s | 42 kB 00:00 2026-03-20T11:20:45.258 INFO:teuthology.orchestra.run.vm07.stdout:(136/138): thrift-0.15.0-4.el9.x86_64.rpm 14 MB/s | 1.6 MB 00:00 2026-03-20T11:20:45.302 INFO:teuthology.orchestra.run.vm08.stdout:(17/138): librgw2-20.2.0-712.g70f8415b.el9.x86_ 4.1 MB/s | 6.4 MB 00:01 2026-03-20T11:20:45.318 INFO:teuthology.orchestra.run.vm05.stdout:(96/138): protobuf-compiler-3.14.0-17.el9.x86_6 982 kB/s | 862 kB 00:00 2026-03-20T11:20:45.399 INFO:teuthology.orchestra.run.vm05.stdout:(97/138): libarrow-doc-9.0.0-15.el9.noarch.rpm 309 kB/s | 25 kB 00:00 2026-03-20T11:20:45.413 INFO:teuthology.orchestra.run.vm08.stdout:(18/138): python3-ceph-argparse-20.2.0-712.g70f 409 kB/s | 45 kB 00:00 2026-03-20T11:20:45.453 INFO:teuthology.orchestra.run.vm05.stdout:(98/138): liboath-2.6.12-1.el9.x86_64.rpm 912 kB/s | 49 kB 00:00 2026-03-20T11:20:45.482 INFO:teuthology.orchestra.run.vm05.stdout:(99/138): libunwind-1.6.2-1.el9.x86_64.rpm 2.3 MB/s | 67 kB 00:00 2026-03-20T11:20:45.498 INFO:teuthology.orchestra.run.vm05.stdout:(100/138): libarrow-9.0.0-15.el9.x86_64.rpm 13 MB/s | 4.4 MB 00:00 2026-03-20T11:20:45.524 INFO:teuthology.orchestra.run.vm08.stdout:(19/138): python3-ceph-common-20.2.0-712.g70f84 1.5 MB/s | 175 kB 00:00 2026-03-20T11:20:45.525 INFO:teuthology.orchestra.run.vm02.stdout:(86/138): qatlib-25.08.0-2.el9.x86_64.rpm 819 kB/s | 240 kB 00:00 2026-03-20T11:20:45.536 INFO:teuthology.orchestra.run.vm05.stdout:(101/138): luarocks-3.9.2-5.el9.noarch.rpm 2.7 MB/s | 151 kB 00:00 2026-03-20T11:20:45.595 INFO:teuthology.orchestra.run.vm05.stdout:(102/138): python3-asyncssh-2.13.2-5.el9.noarch 9.1 MB/s | 548 kB 00:00 2026-03-20T11:20:45.605 INFO:teuthology.orchestra.run.vm05.stdout:(103/138): parquet-libs-9.0.0-15.el9.x86_64.rpm 7.7 MB/s | 838 kB 00:00 2026-03-20T11:20:45.623 INFO:teuthology.orchestra.run.vm05.stdout:(104/138): python3-autocommand-2.2.2-8.el9.noar 1.0 MB/s | 29 kB 00:00 2026-03-20T11:20:45.630 INFO:teuthology.orchestra.run.vm02.stdout:(87/138): qatlib-service-25.08.0-2.el9.x86_64.r 358 kB/s | 37 kB 00:00 2026-03-20T11:20:45.634 INFO:teuthology.orchestra.run.vm05.stdout:(105/138): python3-backports-tarfile-1.2.0-1.el 2.0 MB/s | 60 kB 00:00 2026-03-20T11:20:45.636 INFO:teuthology.orchestra.run.vm08.stdout:(20/138): python3-cephfs-20.2.0-712.g70f8415b.e 1.4 MB/s | 163 kB 00:00 2026-03-20T11:20:45.652 INFO:teuthology.orchestra.run.vm05.stdout:(106/138): python3-bcrypt-3.2.2-1.el9.x86_64.rp 1.5 MB/s | 43 kB 00:00 2026-03-20T11:20:45.662 INFO:teuthology.orchestra.run.vm05.stdout:(107/138): python3-cachetools-4.2.4-1.el9.noarc 1.1 MB/s | 32 kB 00:00 2026-03-20T11:20:45.679 INFO:teuthology.orchestra.run.vm05.stdout:(108/138): python3-certifi-2023.05.07-4.el9.noa 521 kB/s | 14 kB 00:00 2026-03-20T11:20:45.692 INFO:teuthology.orchestra.run.vm05.stdout:(109/138): python3-cheroot-10.0.1-4.el9.noarch. 5.6 MB/s | 173 kB 00:00 2026-03-20T11:20:45.711 INFO:teuthology.orchestra.run.vm05.stdout:(110/138): python3-cherrypy-18.6.1-2.el9.noarch 11 MB/s | 358 kB 00:00 2026-03-20T11:20:45.723 INFO:teuthology.orchestra.run.vm05.stdout:(111/138): python3-google-auth-2.45.0-1.el9.noa 8.1 MB/s | 254 kB 00:00 2026-03-20T11:20:45.736 INFO:teuthology.orchestra.run.vm02.stdout:(88/138): qatzip-libs-1.3.1-1.el9.x86_64.rpm 627 kB/s | 66 kB 00:00 2026-03-20T11:20:45.749 INFO:teuthology.orchestra.run.vm08.stdout:(21/138): python3-rados-20.2.0-712.g70f8415b.el 2.8 MB/s | 324 kB 00:00 2026-03-20T11:20:45.759 INFO:teuthology.orchestra.run.vm05.stdout:(112/138): python3-grpcio-tools-1.46.7-10.el9.x 4.0 MB/s | 144 kB 00:00 2026-03-20T11:20:45.790 INFO:teuthology.orchestra.run.vm05.stdout:(113/138): python3-jaraco-8.2.1-3.el9.noarch.rp 350 kB/s | 11 kB 00:00 2026-03-20T11:20:45.819 INFO:teuthology.orchestra.run.vm05.stdout:(114/138): python3-jaraco-classes-3.2.1-5.el9.n 603 kB/s | 18 kB 00:00 2026-03-20T11:20:45.847 INFO:teuthology.orchestra.run.vm05.stdout:(115/138): python3-jaraco-collections-3.0.0-8.e 850 kB/s | 23 kB 00:00 2026-03-20T11:20:45.864 INFO:teuthology.orchestra.run.vm08.stdout:(22/138): python3-rbd-20.2.0-712.g70f8415b.el9. 2.6 MB/s | 304 kB 00:00 2026-03-20T11:20:45.875 INFO:teuthology.orchestra.run.vm05.stdout:(116/138): python3-jaraco-context-6.0.1-3.el9.n 718 kB/s | 20 kB 00:00 2026-03-20T11:20:45.878 INFO:teuthology.orchestra.run.vm02.stdout:(89/138): socat-1.7.4.1-8.el9.x86_64.rpm 2.1 MB/s | 303 kB 00:00 2026-03-20T11:20:45.902 INFO:teuthology.orchestra.run.vm05.stdout:(117/138): python3-jaraco-functools-3.5.0-2.el9 716 kB/s | 19 kB 00:00 2026-03-20T11:20:45.931 INFO:teuthology.orchestra.run.vm05.stdout:(118/138): python3-jaraco-text-4.0.0-2.el9.noar 930 kB/s | 26 kB 00:00 2026-03-20T11:20:45.949 INFO:teuthology.orchestra.run.vm02.stdout:(90/138): python3-scipy-1.9.3-2.el9.x86_64.rpm 18 MB/s | 19 MB 00:01 2026-03-20T11:20:45.977 INFO:teuthology.orchestra.run.vm08.stdout:(23/138): python3-rgw-20.2.0-712.g70f8415b.el9. 885 kB/s | 99 kB 00:00 2026-03-20T11:20:45.990 INFO:teuthology.orchestra.run.vm02.stdout:(91/138): xmlstarlet-1.6.1-20.el9.x86_64.rpm 575 kB/s | 64 kB 00:00 2026-03-20T11:20:46.013 INFO:teuthology.orchestra.run.vm05.stdout:(119/138): python3-grpcio-1.46.7-10.el9.x86_64. 6.8 MB/s | 2.0 MB 00:00 2026-03-20T11:20:46.024 INFO:teuthology.orchestra.run.vm05.stdout:(120/138): python3-kubernetes-26.1.0-3.el9.noar 11 MB/s | 1.0 MB 00:00 2026-03-20T11:20:46.041 INFO:teuthology.orchestra.run.vm05.stdout:(121/138): python3-more-itertools-8.12.0-2.el9. 2.7 MB/s | 79 kB 00:00 2026-03-20T11:20:46.052 INFO:teuthology.orchestra.run.vm05.stdout:(122/138): python3-natsort-7.1.1-5.el9.noarch.r 2.1 MB/s | 58 kB 00:00 2026-03-20T11:20:46.061 INFO:teuthology.orchestra.run.vm02.stdout:(92/138): lua-devel-5.4.4-4.el9.x86_64.rpm 200 kB/s | 22 kB 00:00 2026-03-20T11:20:46.069 INFO:teuthology.orchestra.run.vm05.stdout:(123/138): python3-portend-3.1.0-2.el9.noarch.r 607 kB/s | 16 kB 00:00 2026-03-20T11:20:46.080 INFO:teuthology.orchestra.run.vm05.stdout:(124/138): python3-pyOpenSSL-21.0.0-1.el9.noarc 3.1 MB/s | 90 kB 00:00 2026-03-20T11:20:46.087 INFO:teuthology.orchestra.run.vm08.stdout:(24/138): rbd-fuse-20.2.0-712.g70f8415b.el9.x86 826 kB/s | 91 kB 00:00 2026-03-20T11:20:46.097 INFO:teuthology.orchestra.run.vm05.stdout:(125/138): python3-repoze-lru-0.7-16.el9.noarch 1.0 MB/s | 31 kB 00:00 2026-03-20T11:20:46.111 INFO:teuthology.orchestra.run.vm05.stdout:(126/138): python3-routes-2.5.1-5.el9.noarch.rp 6.0 MB/s | 188 kB 00:00 2026-03-20T11:20:46.126 INFO:teuthology.orchestra.run.vm05.stdout:(127/138): python3-rsa-4.9-2.el9.noarch.rpm 2.0 MB/s | 59 kB 00:00 2026-03-20T11:20:46.139 INFO:teuthology.orchestra.run.vm05.stdout:(128/138): python3-tempora-5.0.0-2.el9.noarch.r 1.3 MB/s | 36 kB 00:00 2026-03-20T11:20:46.155 INFO:teuthology.orchestra.run.vm05.stdout:(129/138): python3-typing-extensions-4.15.0-1.e 3.0 MB/s | 86 kB 00:00 2026-03-20T11:20:46.167 INFO:teuthology.orchestra.run.vm05.stdout:(130/138): python3-websocket-client-1.2.3-2.el9 3.1 MB/s | 90 kB 00:00 2026-03-20T11:20:46.182 INFO:teuthology.orchestra.run.vm05.stdout:(131/138): python3-xmltodict-0.12.0-15.el9.noar 819 kB/s | 22 kB 00:00 2026-03-20T11:20:46.195 INFO:teuthology.orchestra.run.vm07.stdout:(137/138): librbd1-20.2.0-712.g70f8415b.el9.x86 2.8 MB/s | 2.8 MB 00:01 2026-03-20T11:20:46.197 INFO:teuthology.orchestra.run.vm05.stdout:(132/138): python3-zc-lockfile-2.0-10.el9.noarc 679 kB/s | 20 kB 00:00 2026-03-20T11:20:46.213 INFO:teuthology.orchestra.run.vm05.stdout:(133/138): re2-20211101-20.el9.x86_64.rpm 6.1 MB/s | 191 kB 00:00 2026-03-20T11:20:46.226 INFO:teuthology.orchestra.run.vm07.stdout:(138/138): librados2-20.2.0-712.g70f8415b.el9.x 3.3 MB/s | 3.5 MB 00:01 2026-03-20T11:20:46.228 INFO:teuthology.orchestra.run.vm05.stdout:(134/138): s3cmd-2.4.0-1.el9.noarch.rpm 6.5 MB/s | 206 kB 00:00 2026-03-20T11:20:46.231 INFO:teuthology.orchestra.run.vm07.stdout:-------------------------------------------------------------------------------- 2026-03-20T11:20:46.231 INFO:teuthology.orchestra.run.vm07.stdout:Total 15 MB/s | 267 MB 00:17 2026-03-20T11:20:46.577 INFO:teuthology.orchestra.run.vm05.stdout:(135/138): thrift-0.15.0-4.el9.x86_64.rpm 4.4 MB/s | 1.6 MB 00:00 2026-03-20T11:20:46.674 INFO:teuthology.orchestra.run.vm02.stdout:(93/138): protobuf-compiler-3.14.0-17.el9.x86_6 1.2 MB/s | 862 kB 00:00 2026-03-20T11:20:46.701 INFO:teuthology.orchestra.run.vm05.stdout:(136/138): python3-scipy-1.9.3-2.el9.x86_64.rpm 6.6 MB/s | 19 MB 00:02 2026-03-20T11:20:46.748 INFO:teuthology.orchestra.run.vm08.stdout:(25/138): rbd-mirror-20.2.0-712.g70f8415b.el9.x 4.4 MB/s | 2.9 MB 00:00 2026-03-20T11:20:46.860 INFO:teuthology.orchestra.run.vm08.stdout:(26/138): rbd-nbd-20.2.0-712.g70f8415b.el9.x86_ 1.6 MB/s | 180 kB 00:00 2026-03-20T11:20:46.861 INFO:teuthology.orchestra.run.vm02.stdout:(94/138): abseil-cpp-20211102.0-4.el9.x86_64.rp 690 kB/s | 551 kB 00:00 2026-03-20T11:20:46.938 INFO:teuthology.orchestra.run.vm02.stdout:(95/138): gperftools-libs-2.9.1-3.el9.x86_64.rp 1.1 MB/s | 308 kB 00:00 2026-03-20T11:20:46.954 INFO:teuthology.orchestra.run.vm08.stdout:(27/138): ceph-osd-20.2.0-712.g70f8415b.el9.x86 1.8 MB/s | 17 MB 00:09 2026-03-20T11:20:46.954 INFO:teuthology.orchestra.run.vm02.stdout:(96/138): grpc-data-1.46.7-10.el9.noarch.rpm 209 kB/s | 19 kB 00:00 2026-03-20T11:20:46.969 INFO:teuthology.orchestra.run.vm08.stdout:(28/138): ceph-grafana-dashboards-20.2.0-712.g7 397 kB/s | 43 kB 00:00 2026-03-20T11:20:46.972 INFO:teuthology.orchestra.run.vm07.stdout:Running transaction check 2026-03-20T11:20:47.041 INFO:teuthology.orchestra.run.vm07.stdout:Transaction check succeeded. 2026-03-20T11:20:47.041 INFO:teuthology.orchestra.run.vm07.stdout:Running transaction test 2026-03-20T11:20:47.070 INFO:teuthology.orchestra.run.vm08.stdout:(29/138): ceph-mgr-cephadm-20.2.0-712.g70f8415b 1.5 MB/s | 173 kB 00:00 2026-03-20T11:20:47.266 INFO:teuthology.orchestra.run.vm02.stdout:(97/138): libarrow-doc-9.0.0-15.el9.noarch.rpm 80 kB/s | 25 kB 00:00 2026-03-20T11:20:47.362 INFO:teuthology.orchestra.run.vm02.stdout:(98/138): libarrow-9.0.0-15.el9.x86_64.rpm 10 MB/s | 4.4 MB 00:00 2026-03-20T11:20:47.363 INFO:teuthology.orchestra.run.vm02.stdout:(99/138): liboath-2.6.12-1.el9.x86_64.rpm 504 kB/s | 49 kB 00:00 2026-03-20T11:20:47.448 INFO:teuthology.orchestra.run.vm02.stdout:(100/138): libunwind-1.6.2-1.el9.x86_64.rpm 807 kB/s | 67 kB 00:00 2026-03-20T11:20:47.463 INFO:teuthology.orchestra.run.vm02.stdout:(101/138): luarocks-3.9.2-5.el9.noarch.rpm 1.5 MB/s | 151 kB 00:00 2026-03-20T11:20:47.508 INFO:teuthology.orchestra.run.vm05.stdout:(137/138): librados2-20.2.0-712.g70f8415b.el9.x 2.8 MB/s | 3.5 MB 00:01 2026-03-20T11:20:47.577 INFO:teuthology.orchestra.run.vm02.stdout:(102/138): parquet-libs-9.0.0-15.el9.x86_64.rpm 6.3 MB/s | 838 kB 00:00 2026-03-20T11:20:47.619 INFO:teuthology.orchestra.run.vm02.stdout:(103/138): python3-asyncssh-2.13.2-5.el9.noarch 3.4 MB/s | 548 kB 00:00 2026-03-20T11:20:47.677 INFO:teuthology.orchestra.run.vm05.stdout:(138/138): librbd1-20.2.0-712.g70f8415b.el9.x86 2.6 MB/s | 2.8 MB 00:01 2026-03-20T11:20:47.680 INFO:teuthology.orchestra.run.vm05.stdout:-------------------------------------------------------------------------------- 2026-03-20T11:20:47.680 INFO:teuthology.orchestra.run.vm05.stdout:Total 14 MB/s | 267 MB 00:18 2026-03-20T11:20:47.690 INFO:teuthology.orchestra.run.vm02.stdout:(104/138): python3-autocommand-2.2.2-8.el9.noar 262 kB/s | 29 kB 00:00 2026-03-20T11:20:47.717 INFO:teuthology.orchestra.run.vm02.stdout:(105/138): python3-backports-tarfile-1.2.0-1.el 613 kB/s | 60 kB 00:00 2026-03-20T11:20:47.792 INFO:teuthology.orchestra.run.vm02.stdout:(106/138): python3-bcrypt-3.2.2-1.el9.x86_64.rp 425 kB/s | 43 kB 00:00 2026-03-20T11:20:47.824 INFO:teuthology.orchestra.run.vm02.stdout:(107/138): python3-cachetools-4.2.4-1.el9.noarc 302 kB/s | 32 kB 00:00 2026-03-20T11:20:47.907 INFO:teuthology.orchestra.run.vm02.stdout:(108/138): python3-certifi-2023.05.07-4.el9.noa 124 kB/s | 14 kB 00:00 2026-03-20T11:20:47.912 INFO:teuthology.orchestra.run.vm02.stdout:(109/138): python3-cheroot-10.0.1-4.el9.noarch. 1.9 MB/s | 173 kB 00:00 2026-03-20T11:20:47.994 INFO:teuthology.orchestra.run.vm02.stdout:(110/138): python3-cherrypy-18.6.1-2.el9.noarch 4.0 MB/s | 358 kB 00:00 2026-03-20T11:20:48.019 INFO:teuthology.orchestra.run.vm02.stdout:(111/138): python3-google-auth-2.45.0-1.el9.noa 2.3 MB/s | 254 kB 00:00 2026-03-20T11:20:48.232 INFO:teuthology.orchestra.run.vm07.stdout:Transaction test succeeded. 2026-03-20T11:20:48.232 INFO:teuthology.orchestra.run.vm07.stdout:Running transaction 2026-03-20T11:20:48.307 INFO:teuthology.orchestra.run.vm02.stdout:(112/138): python3-grpcio-1.46.7-10.el9.x86_64. 6.5 MB/s | 2.0 MB 00:00 2026-03-20T11:20:48.309 INFO:teuthology.orchestra.run.vm02.stdout:(113/138): python3-grpcio-tools-1.46.7-10.el9.x 498 kB/s | 144 kB 00:00 2026-03-20T11:20:48.412 INFO:teuthology.orchestra.run.vm02.stdout:(114/138): python3-jaraco-8.2.1-3.el9.noarch.rp 105 kB/s | 11 kB 00:00 2026-03-20T11:20:48.437 INFO:teuthology.orchestra.run.vm02.stdout:(115/138): python3-jaraco-classes-3.2.1-5.el9.n 138 kB/s | 18 kB 00:00 2026-03-20T11:20:48.462 INFO:teuthology.orchestra.run.vm05.stdout:Running transaction check 2026-03-20T11:20:48.495 INFO:teuthology.orchestra.run.vm02.stdout:(116/138): python3-jaraco-collections-3.0.0-8.e 279 kB/s | 23 kB 00:00 2026-03-20T11:20:48.519 INFO:teuthology.orchestra.run.vm02.stdout:(117/138): python3-jaraco-context-6.0.1-3.el9.n 241 kB/s | 20 kB 00:00 2026-03-20T11:20:48.555 INFO:teuthology.orchestra.run.vm05.stdout:Transaction check succeeded. 2026-03-20T11:20:48.555 INFO:teuthology.orchestra.run.vm05.stdout:Running transaction test 2026-03-20T11:20:48.577 INFO:teuthology.orchestra.run.vm02.stdout:(118/138): python3-jaraco-functools-3.5.0-2.el9 238 kB/s | 19 kB 00:00 2026-03-20T11:20:48.602 INFO:teuthology.orchestra.run.vm02.stdout:(119/138): python3-jaraco-text-4.0.0-2.el9.noar 319 kB/s | 26 kB 00:00 2026-03-20T11:20:48.719 INFO:teuthology.orchestra.run.vm02.stdout:(120/138): python3-kubernetes-26.1.0-3.el9.noar 7.2 MB/s | 1.0 MB 00:00 2026-03-20T11:20:48.721 INFO:teuthology.orchestra.run.vm02.stdout:(121/138): python3-more-itertools-8.12.0-2.el9. 667 kB/s | 79 kB 00:00 2026-03-20T11:20:48.817 INFO:teuthology.orchestra.run.vm02.stdout:(122/138): python3-natsort-7.1.1-5.el9.noarch.r 600 kB/s | 58 kB 00:00 2026-03-20T11:20:48.819 INFO:teuthology.orchestra.run.vm02.stdout:(123/138): python3-portend-3.1.0-2.el9.noarch.r 168 kB/s | 16 kB 00:00 2026-03-20T11:20:48.901 INFO:teuthology.orchestra.run.vm02.stdout:(124/138): python3-pyOpenSSL-21.0.0-1.el9.noarc 1.1 MB/s | 90 kB 00:00 2026-03-20T11:20:48.923 INFO:teuthology.orchestra.run.vm02.stdout:(125/138): python3-repoze-lru-0.7-16.el9.noarch 295 kB/s | 31 kB 00:00 2026-03-20T11:20:49.014 INFO:teuthology.orchestra.run.vm02.stdout:(126/138): python3-routes-2.5.1-5.el9.noarch.rp 1.6 MB/s | 188 kB 00:00 2026-03-20T11:20:49.016 INFO:teuthology.orchestra.run.vm02.stdout:(127/138): python3-rsa-4.9-2.el9.noarch.rpm 640 kB/s | 59 kB 00:00 2026-03-20T11:20:49.099 INFO:teuthology.orchestra.run.vm02.stdout:(128/138): python3-tempora-5.0.0-2.el9.noarch.r 430 kB/s | 36 kB 00:00 2026-03-20T11:20:49.101 INFO:teuthology.orchestra.run.vm02.stdout:(129/138): python3-typing-extensions-4.15.0-1.e 1.0 MB/s | 86 kB 00:00 2026-03-20T11:20:49.206 INFO:teuthology.orchestra.run.vm02.stdout:(130/138): python3-websocket-client-1.2.3-2.el9 842 kB/s | 90 kB 00:00 2026-03-20T11:20:49.223 INFO:teuthology.orchestra.run.vm02.stdout:(131/138): python3-xmltodict-0.12.0-15.el9.noar 182 kB/s | 22 kB 00:00 2026-03-20T11:20:49.288 INFO:teuthology.orchestra.run.vm02.stdout:(132/138): python3-zc-lockfile-2.0-10.el9.noarc 245 kB/s | 20 kB 00:00 2026-03-20T11:20:49.308 INFO:teuthology.orchestra.run.vm02.stdout:(133/138): re2-20211101-20.el9.x86_64.rpm 2.2 MB/s | 191 kB 00:00 2026-03-20T11:20:49.402 INFO:teuthology.orchestra.run.vm02.stdout:(134/138): s3cmd-2.4.0-1.el9.noarch.rpm 1.8 MB/s | 206 kB 00:00 2026-03-20T11:20:49.512 INFO:teuthology.orchestra.run.vm02.stdout:(135/138): thrift-0.15.0-4.el9.x86_64.rpm 7.8 MB/s | 1.6 MB 00:00 2026-03-20T11:20:49.620 INFO:teuthology.orchestra.run.vm07.stdout: Preparing : 1/1 2026-03-20T11:20:49.631 INFO:teuthology.orchestra.run.vm07.stdout: Installing : thrift-0.15.0-4.el9.x86_64 1/140 2026-03-20T11:20:49.635 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 2/140 2026-03-20T11:20:49.653 INFO:teuthology.orchestra.run.vm07.stdout: Installing : liboath-2.6.12-1.el9.x86_64 3/140 2026-03-20T11:20:49.758 INFO:teuthology.orchestra.run.vm05.stdout:Transaction test succeeded. 2026-03-20T11:20:49.759 INFO:teuthology.orchestra.run.vm05.stdout:Running transaction 2026-03-20T11:20:49.868 INFO:teuthology.orchestra.run.vm07.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 4/140 2026-03-20T11:20:49.870 INFO:teuthology.orchestra.run.vm07.stdout: Upgrading : librados2-2:20.2.0-712.g70f8415b.el9.x86_64 5/140 2026-03-20T11:20:49.928 INFO:teuthology.orchestra.run.vm08.stdout:(30/138): ceph-mgr-dashboard-20.2.0-712.g70f841 3.6 MB/s | 11 MB 00:02 2026-03-20T11:20:49.950 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: librados2-2:20.2.0-712.g70f8415b.el9.x86_64 5/140 2026-03-20T11:20:49.962 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-rados-2:20.2.0-712.g70f8415b.el9.x86_64 6/140 2026-03-20T11:20:49.967 INFO:teuthology.orchestra.run.vm07.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 7/140 2026-03-20T11:20:49.972 INFO:teuthology.orchestra.run.vm07.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 8/140 2026-03-20T11:20:49.978 INFO:teuthology.orchestra.run.vm07.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 9/140 2026-03-20T11:20:50.040 INFO:teuthology.orchestra.run.vm08.stdout:(31/138): ceph-mgr-modules-core-20.2.0-712.g70f 2.5 MB/s | 290 kB 00:00 2026-03-20T11:20:50.045 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 10/140 2026-03-20T11:20:50.150 INFO:teuthology.orchestra.run.vm08.stdout:(32/138): ceph-mgr-rook-20.2.0-712.g70f8415b.el 457 kB/s | 50 kB 00:00 2026-03-20T11:20:50.204 INFO:teuthology.orchestra.run.vm07.stdout: Installing : libnbd-1.20.3-4.el9.x86_64 11/140 2026-03-20T11:20:50.207 INFO:teuthology.orchestra.run.vm07.stdout: Upgrading : librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 12/140 2026-03-20T11:20:50.233 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 12/140 2026-03-20T11:20:50.236 INFO:teuthology.orchestra.run.vm07.stdout: Installing : libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 13/140 2026-03-20T11:20:50.259 INFO:teuthology.orchestra.run.vm08.stdout:(33/138): ceph-prometheus-alerts-20.2.0-712.g70 160 kB/s | 17 kB 00:00 2026-03-20T11:20:50.266 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 13/140 2026-03-20T11:20:50.269 INFO:teuthology.orchestra.run.vm07.stdout: Installing : libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_ 14/140 2026-03-20T11:20:50.291 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_ 14/140 2026-03-20T11:20:50.339 INFO:teuthology.orchestra.run.vm07.stdout: Installing : re2-1:20211101-20.el9.x86_64 15/140 2026-03-20T11:20:50.371 INFO:teuthology.orchestra.run.vm08.stdout:(34/138): ceph-volume-20.2.0-712.g70f8415b.el9. 2.6 MB/s | 298 kB 00:00 2026-03-20T11:20:50.376 INFO:teuthology.orchestra.run.vm07.stdout: Installing : libarrow-9.0.0-15.el9.x86_64 16/140 2026-03-20T11:20:50.389 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-pyasn1-0.4.8-7.el9.noarch 17/140 2026-03-20T11:20:50.397 INFO:teuthology.orchestra.run.vm07.stdout: Installing : protobuf-3.14.0-17.el9.x86_64 18/140 2026-03-20T11:20:50.403 INFO:teuthology.orchestra.run.vm07.stdout: Installing : lua-5.4.4-4.el9.x86_64 19/140 2026-03-20T11:20:50.412 INFO:teuthology.orchestra.run.vm07.stdout: Installing : flexiblas-3.0.4-9.el9.x86_64 20/140 2026-03-20T11:20:50.451 INFO:teuthology.orchestra.run.vm07.stdout: Installing : unzip-6.0-59.el9.x86_64 21/140 2026-03-20T11:20:50.471 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-urllib3-1.26.5-7.el9.noarch 22/140 2026-03-20T11:20:50.476 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-requests-2.25.1-10.el9.noarch 23/140 2026-03-20T11:20:50.485 INFO:teuthology.orchestra.run.vm07.stdout: Installing : libquadmath-11.5.0-14.el9.x86_64 24/140 2026-03-20T11:20:50.487 INFO:teuthology.orchestra.run.vm07.stdout: Installing : libgfortran-11.5.0-14.el9.x86_64 25/140 2026-03-20T11:20:50.533 INFO:teuthology.orchestra.run.vm07.stdout: Installing : ledmon-libs-1.1.0-3.el9.x86_64 26/140 2026-03-20T11:20:50.542 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-ceph-common-2:20.2.0-712.g70f8415b.el9.x 27/140 2026-03-20T11:20:50.545 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-ceph-argparse-2:20.2.0-712.g70f8415b.el9 28/140 2026-03-20T11:20:50.546 INFO:teuthology.orchestra.run.vm07.stdout: Installing : libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_ 29/140 2026-03-20T11:20:50.609 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_ 29/140 2026-03-20T11:20:50.612 INFO:teuthology.orchestra.run.vm07.stdout: Installing : libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 30/140 2026-03-20T11:20:50.637 INFO:teuthology.orchestra.run.vm02.stdout:(136/138): librbd1-20.2.0-712.g70f8415b.el9.x86 2.5 MB/s | 2.8 MB 00:01 2026-03-20T11:20:50.642 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 30/140 2026-03-20T11:20:50.661 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-cephfs-2:20.2.0-712.g70f8415b.el9.x86_64 31/140 2026-03-20T11:20:50.786 INFO:teuthology.orchestra.run.vm08.stdout:(35/138): cephadm-20.2.0-712.g70f8415b.el9.noar 2.4 MB/s | 1.0 MB 00:00 2026-03-20T11:20:50.786 INFO:teuthology.orchestra.run.vm02.stdout:(137/138): librados2-20.2.0-712.g70f8415b.el9.x 2.5 MB/s | 3.5 MB 00:01 2026-03-20T11:20:50.796 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 32/140 2026-03-20T11:20:50.852 INFO:teuthology.orchestra.run.vm07.stdout: Installing : zip-3.0-35.el9.x86_64 33/140 2026-03-20T11:20:50.903 INFO:teuthology.orchestra.run.vm07.stdout: Installing : luarocks-3.9.2-5.el9.noarch 34/140 2026-03-20T11:20:50.924 INFO:teuthology.orchestra.run.vm07.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 35/140 2026-03-20T11:20:51.010 INFO:teuthology.orchestra.run.vm07.stdout: Installing : protobuf-compiler-3.14.0-17.el9.x86_64 36/140 2026-03-20T11:20:51.013 INFO:teuthology.orchestra.run.vm05.stdout: Preparing : 1/1 2026-03-20T11:20:51.029 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-pyasn1-modules-0.4.8-7.el9.noarch 37/140 2026-03-20T11:20:51.082 INFO:teuthology.orchestra.run.vm05.stdout: Installing : thrift-0.15.0-4.el9.x86_64 1/140 2026-03-20T11:20:51.087 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 2/140 2026-03-20T11:20:51.096 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-rsa-4.9-2.el9.noarch 38/140 2026-03-20T11:20:51.103 INFO:teuthology.orchestra.run.vm05.stdout: Installing : liboath-2.6.12-1.el9.x86_64 3/140 2026-03-20T11:20:51.106 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-rbd-2:20.2.0-712.g70f8415b.el9.x86_64 39/140 2026-03-20T11:20:51.119 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 40/140 2026-03-20T11:20:51.128 INFO:teuthology.orchestra.run.vm07.stdout: Installing : librados-devel-2:20.2.0-712.g70f8415b.el9.x86_64 41/140 2026-03-20T11:20:51.135 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 42/140 2026-03-20T11:20:51.161 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-xmltodict-0.12.0-15.el9.noarch 43/140 2026-03-20T11:20:51.171 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 44/140 2026-03-20T11:20:51.177 INFO:teuthology.orchestra.run.vm08.stdout:(36/138): bzip2-1.0.8-11.el9.x86_64.rpm 140 kB/s | 55 kB 00:00 2026-03-20T11:20:51.182 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-typing-extensions-4.15.0-1.el9.noarch 45/140 2026-03-20T11:20:51.204 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 46/140 2026-03-20T11:20:51.221 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 47/140 2026-03-20T11:20:51.228 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 48/140 2026-03-20T11:20:51.239 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 49/140 2026-03-20T11:20:51.306 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 50/140 2026-03-20T11:20:51.325 INFO:teuthology.orchestra.run.vm08.stdout:(37/138): cryptsetup-2.8.1-3.el9.x86_64.rpm 2.3 MB/s | 351 kB 00:00 2026-03-20T11:20:51.326 INFO:teuthology.orchestra.run.vm05.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 4/140 2026-03-20T11:20:51.330 INFO:teuthology.orchestra.run.vm05.stdout: Upgrading : librados2-2:20.2.0-712.g70f8415b.el9.x86_64 5/140 2026-03-20T11:20:51.369 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: librados2-2:20.2.0-712.g70f8415b.el9.x86_64 5/140 2026-03-20T11:20:51.376 INFO:teuthology.orchestra.run.vm08.stdout:(38/138): fuse-2.9.9-17.el9.x86_64.rpm 1.6 MB/s | 80 kB 00:00 2026-03-20T11:20:51.382 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-rados-2:20.2.0-712.g70f8415b.el9.x86_64 6/140 2026-03-20T11:20:51.388 INFO:teuthology.orchestra.run.vm05.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 7/140 2026-03-20T11:20:51.394 INFO:teuthology.orchestra.run.vm05.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 8/140 2026-03-20T11:20:51.398 INFO:teuthology.orchestra.run.vm05.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 9/140 2026-03-20T11:20:51.406 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 10/140 2026-03-20T11:20:51.426 INFO:teuthology.orchestra.run.vm08.stdout:(39/138): ledmon-libs-1.1.0-3.el9.x86_64.rpm 822 kB/s | 40 kB 00:00 2026-03-20T11:20:51.485 INFO:teuthology.orchestra.run.vm08.stdout:(40/138): libconfig-1.7.2-9.el9.x86_64.rpm 1.2 MB/s | 72 kB 00:00 2026-03-20T11:20:51.585 INFO:teuthology.orchestra.run.vm05.stdout: Installing : libnbd-1.20.3-4.el9.x86_64 11/140 2026-03-20T11:20:51.589 INFO:teuthology.orchestra.run.vm05.stdout: Upgrading : librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 12/140 2026-03-20T11:20:51.616 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 12/140 2026-03-20T11:20:51.618 INFO:teuthology.orchestra.run.vm08.stdout:(41/138): libgfortran-11.5.0-14.el9.x86_64.rpm 5.9 MB/s | 794 kB 00:00 2026-03-20T11:20:51.618 INFO:teuthology.orchestra.run.vm05.stdout: Installing : libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 13/140 2026-03-20T11:20:51.650 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 13/140 2026-03-20T11:20:51.652 INFO:teuthology.orchestra.run.vm05.stdout: Installing : libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_ 14/140 2026-03-20T11:20:51.675 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_ 14/140 2026-03-20T11:20:51.682 INFO:teuthology.orchestra.run.vm08.stdout:(42/138): libquadmath-11.5.0-14.el9.x86_64.rpm 2.8 MB/s | 184 kB 00:00 2026-03-20T11:20:51.717 INFO:teuthology.orchestra.run.vm05.stdout: Installing : re2-1:20211101-20.el9.x86_64 15/140 2026-03-20T11:20:51.732 INFO:teuthology.orchestra.run.vm08.stdout:(43/138): mailcap-2.1.49-5.el9.noarch.rpm 675 kB/s | 33 kB 00:00 2026-03-20T11:20:51.759 INFO:teuthology.orchestra.run.vm05.stdout: Installing : libarrow-9.0.0-15.el9.x86_64 16/140 2026-03-20T11:20:51.771 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-google-auth-1:2.45.0-1.el9.noarch 51/140 2026-03-20T11:20:51.772 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-pyasn1-0.4.8-7.el9.noarch 17/140 2026-03-20T11:20:51.780 INFO:teuthology.orchestra.run.vm05.stdout: Installing : protobuf-3.14.0-17.el9.x86_64 18/140 2026-03-20T11:20:51.785 INFO:teuthology.orchestra.run.vm05.stdout: Installing : lua-5.4.4-4.el9.x86_64 19/140 2026-03-20T11:20:51.789 INFO:teuthology.orchestra.run.vm08.stdout:(44/138): ceph-mgr-diskprediction-local-20.2.0- 1.6 MB/s | 7.4 MB 00:04 2026-03-20T11:20:51.790 INFO:teuthology.orchestra.run.vm08.stdout:(45/138): pciutils-3.7.0-7.el9.x86_64.rpm 1.6 MB/s | 93 kB 00:00 2026-03-20T11:20:51.794 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-kubernetes-1:26.1.0-3.el9.noarch 52/140 2026-03-20T11:20:51.794 INFO:teuthology.orchestra.run.vm05.stdout: Installing : flexiblas-3.0.4-9.el9.x86_64 20/140 2026-03-20T11:20:51.804 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-backports-tarfile-1.2.0-1.el9.noarch 53/140 2026-03-20T11:20:51.817 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-jaraco-context-6.0.1-3.el9.noarch 54/140 2026-03-20T11:20:51.824 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-autocommand-2.2.2-8.el9.noarch 55/140 2026-03-20T11:20:51.828 INFO:teuthology.orchestra.run.vm05.stdout: Installing : unzip-6.0-59.el9.x86_64 21/140 2026-03-20T11:20:51.835 INFO:teuthology.orchestra.run.vm07.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 56/140 2026-03-20T11:20:51.841 INFO:teuthology.orchestra.run.vm07.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 57/140 2026-03-20T11:20:51.841 INFO:teuthology.orchestra.run.vm08.stdout:(46/138): python3-cffi-1.14.5-5.el9.x86_64.rpm 4.8 MB/s | 253 kB 00:00 2026-03-20T11:20:51.844 INFO:teuthology.orchestra.run.vm07.stdout: Installing : libarrow-doc-9.0.0-15.el9.noarch 58/140 2026-03-20T11:20:51.850 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-urllib3-1.26.5-7.el9.noarch 22/140 2026-03-20T11:20:51.856 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-requests-2.25.1-10.el9.noarch 23/140 2026-03-20T11:20:51.866 INFO:teuthology.orchestra.run.vm05.stdout: Installing : libquadmath-11.5.0-14.el9.x86_64 24/140 2026-03-20T11:20:51.870 INFO:teuthology.orchestra.run.vm05.stdout: Installing : libgfortran-11.5.0-14.el9.x86_64 25/140 2026-03-20T11:20:51.884 INFO:teuthology.orchestra.run.vm07.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 59/140 2026-03-20T11:20:51.917 INFO:teuthology.orchestra.run.vm05.stdout: Installing : ledmon-libs-1.1.0-3.el9.x86_64 26/140 2026-03-20T11:20:51.925 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-ceph-common-2:20.2.0-712.g70f8415b.el9.x 27/140 2026-03-20T11:20:51.928 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-ceph-argparse-2:20.2.0-712.g70f8415b.el9 28/140 2026-03-20T11:20:51.929 INFO:teuthology.orchestra.run.vm08.stdout:(47/138): python3-ply-3.11-14.el9.noarch.rpm 1.2 MB/s | 106 kB 00:00 2026-03-20T11:20:51.930 INFO:teuthology.orchestra.run.vm05.stdout: Installing : libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_ 29/140 2026-03-20T11:20:51.950 INFO:teuthology.orchestra.run.vm07.stdout: Installing : abseil-cpp-20211102.0-4.el9.x86_64 60/140 2026-03-20T11:20:51.955 INFO:teuthology.orchestra.run.vm08.stdout:(48/138): python3-cryptography-36.0.1-5.el9.x86 7.6 MB/s | 1.2 MB 00:00 2026-03-20T11:20:51.965 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 61/140 2026-03-20T11:20:51.978 INFO:teuthology.orchestra.run.vm07.stdout: Installing : socat-1.7.4.1-8.el9.x86_64 62/140 2026-03-20T11:20:51.982 INFO:teuthology.orchestra.run.vm08.stdout:(49/138): python3-pycparser-2.20-6.el9.noarch.r 2.5 MB/s | 135 kB 00:00 2026-03-20T11:20:51.985 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 63/140 2026-03-20T11:20:51.990 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_ 29/140 2026-03-20T11:20:51.993 INFO:teuthology.orchestra.run.vm05.stdout: Installing : libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 30/140 2026-03-20T11:20:51.995 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 64/140 2026-03-20T11:20:52.002 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-jaraco-text-4.0.0-2.el9.noarch 65/140 2026-03-20T11:20:52.006 INFO:teuthology.orchestra.run.vm08.stdout:(50/138): python3-pyparsing-2.4.7-9.el9.noarch. 2.9 MB/s | 150 kB 00:00 2026-03-20T11:20:52.013 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 66/140 2026-03-20T11:20:52.023 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 67/140 2026-03-20T11:20:52.029 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 30/140 2026-03-20T11:20:52.033 INFO:teuthology.orchestra.run.vm08.stdout:(51/138): python3-requests-2.25.1-10.el9.noarch 2.5 MB/s | 126 kB 00:00 2026-03-20T11:20:52.052 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-cephfs-2:20.2.0-712.g70f8415b.el9.x86_64 31/140 2026-03-20T11:20:52.057 INFO:teuthology.orchestra.run.vm08.stdout:(52/138): python3-urllib3-1.26.5-7.el9.noarch.r 4.2 MB/s | 218 kB 00:00 2026-03-20T11:20:52.064 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 32/140 2026-03-20T11:20:52.075 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 68/140 2026-03-20T11:20:52.085 INFO:teuthology.orchestra.run.vm08.stdout:(53/138): unzip-6.0-59.el9.x86_64.rpm 3.4 MB/s | 182 kB 00:00 2026-03-20T11:20:52.096 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-protobuf-3.14.0-17.el9.noarch 69/140 2026-03-20T11:20:52.103 INFO:teuthology.orchestra.run.vm05.stdout: Installing : zip-3.0-35.el9.x86_64 33/140 2026-03-20T11:20:52.109 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 70/140 2026-03-20T11:20:52.109 INFO:teuthology.orchestra.run.vm05.stdout: Installing : luarocks-3.9.2-5.el9.noarch 34/140 2026-03-20T11:20:52.109 INFO:teuthology.orchestra.run.vm08.stdout:(54/138): zip-3.0-35.el9.x86_64.rpm 5.0 MB/s | 266 kB 00:00 2026-03-20T11:20:52.119 INFO:teuthology.orchestra.run.vm05.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 35/140 2026-03-20T11:20:52.124 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 71/140 2026-03-20T11:20:52.178 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-jmespath-1.0.1-1.el9.noarch 72/140 2026-03-20T11:20:52.194 INFO:teuthology.orchestra.run.vm05.stdout: Installing : protobuf-compiler-3.14.0-17.el9.x86_64 36/140 2026-03-20T11:20:52.215 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-pyasn1-modules-0.4.8-7.el9.noarch 37/140 2026-03-20T11:20:52.227 INFO:teuthology.orchestra.run.vm08.stdout:(55/138): flexiblas-3.0.4-9.el9.x86_64.rpm 253 kB/s | 30 kB 00:00 2026-03-20T11:20:52.239 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-rsa-4.9-2.el9.noarch 38/140 2026-03-20T11:20:52.248 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-rbd-2:20.2.0-712.g70f8415b.el9.x86_64 39/140 2026-03-20T11:20:52.262 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 40/140 2026-03-20T11:20:52.271 INFO:teuthology.orchestra.run.vm05.stdout: Installing : librados-devel-2:20.2.0-712.g70f8415b.el9.x86_64 41/140 2026-03-20T11:20:52.279 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 42/140 2026-03-20T11:20:52.303 INFO:teuthology.orchestra.run.vm08.stdout:(56/138): boost-program-options-1.75.0-13.el9.x 478 kB/s | 104 kB 00:00 2026-03-20T11:20:52.308 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-xmltodict-0.12.0-15.el9.noarch 43/140 2026-03-20T11:20:52.316 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 44/140 2026-03-20T11:20:52.328 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-typing-extensions-4.15.0-1.el9.noarch 45/140 2026-03-20T11:20:52.343 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 46/140 2026-03-20T11:20:52.349 INFO:teuthology.orchestra.run.vm08.stdout:(57/138): flexiblas-openblas-openmp-3.0.4-9.el9 329 kB/s | 15 kB 00:00 2026-03-20T11:20:52.358 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 47/140 2026-03-20T11:20:52.367 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 48/140 2026-03-20T11:20:52.383 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 49/140 2026-03-20T11:20:52.436 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 50/140 2026-03-20T11:20:52.443 INFO:teuthology.orchestra.run.vm08.stdout:(58/138): libnbd-1.20.3-4.el9.x86_64.rpm 1.7 MB/s | 164 kB 00:00 2026-03-20T11:20:52.512 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-devel-3.9.25-3.el9.x86_64 73/140 2026-03-20T11:20:52.542 INFO:teuthology.orchestra.run.vm08.stdout:(59/138): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.6 MB/s | 160 kB 00:00 2026-03-20T11:20:52.550 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 74/140 2026-03-20T11:20:52.555 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-jinja2-2.11.3-8.el9.noarch 75/140 2026-03-20T11:20:52.562 INFO:teuthology.orchestra.run.vm07.stdout: Installing : perl-Benchmark-1.23-483.el9.noarch 76/140 2026-03-20T11:20:52.611 INFO:teuthology.orchestra.run.vm08.stdout:(60/138): librabbitmq-0.11.0-7.el9.x86_64.rpm 657 kB/s | 45 kB 00:00 2026-03-20T11:20:52.640 INFO:teuthology.orchestra.run.vm07.stdout: Installing : openblas-0.3.29-1.el9.x86_64 77/140 2026-03-20T11:20:52.643 INFO:teuthology.orchestra.run.vm07.stdout: Installing : openblas-openmp-0.3.29-1.el9.x86_64 78/140 2026-03-20T11:20:52.676 INFO:teuthology.orchestra.run.vm07.stdout: Installing : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 79/140 2026-03-20T11:20:52.821 INFO:teuthology.orchestra.run.vm08.stdout:(61/138): librdkafka-1.6.1-102.el9.x86_64.rpm 3.1 MB/s | 662 kB 00:00 2026-03-20T11:20:52.904 INFO:teuthology.orchestra.run.vm08.stdout:(62/138): libstoragemgmt-1.10.1-1.el9.x86_64.rp 2.9 MB/s | 246 kB 00:00 2026-03-20T11:20:52.936 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-google-auth-1:2.45.0-1.el9.noarch 51/140 2026-03-20T11:20:52.958 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-kubernetes-1:26.1.0-3.el9.noarch 52/140 2026-03-20T11:20:52.966 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-backports-tarfile-1.2.0-1.el9.noarch 53/140 2026-03-20T11:20:52.979 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-jaraco-context-6.0.1-3.el9.noarch 54/140 2026-03-20T11:20:52.984 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-autocommand-2.2.2-8.el9.noarch 55/140 2026-03-20T11:20:52.995 INFO:teuthology.orchestra.run.vm05.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 56/140 2026-03-20T11:20:53.000 INFO:teuthology.orchestra.run.vm08.stdout:(63/138): libxslt-1.1.34-12.el9.x86_64.rpm 2.4 MB/s | 233 kB 00:00 2026-03-20T11:20:53.002 INFO:teuthology.orchestra.run.vm05.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 57/140 2026-03-20T11:20:53.005 INFO:teuthology.orchestra.run.vm05.stdout: Installing : libarrow-doc-9.0.0-15.el9.noarch 58/140 2026-03-20T11:20:53.045 INFO:teuthology.orchestra.run.vm05.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 59/140 2026-03-20T11:20:53.100 INFO:teuthology.orchestra.run.vm08.stdout:(64/138): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.9 MB/s | 292 kB 00:00 2026-03-20T11:20:53.110 INFO:teuthology.orchestra.run.vm05.stdout: Installing : abseil-cpp-20211102.0-4.el9.x86_64 60/140 2026-03-20T11:20:53.128 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 61/140 2026-03-20T11:20:53.139 INFO:teuthology.orchestra.run.vm05.stdout: Installing : socat-1.7.4.1-8.el9.x86_64 62/140 2026-03-20T11:20:53.146 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 63/140 2026-03-20T11:20:53.156 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 64/140 2026-03-20T11:20:53.161 INFO:teuthology.orchestra.run.vm07.stdout: Installing : flexiblas-netlib-3.0.4-9.el9.x86_64 80/140 2026-03-20T11:20:53.163 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-jaraco-text-4.0.0-2.el9.noarch 65/140 2026-03-20T11:20:53.166 INFO:teuthology.orchestra.run.vm08.stdout:(65/138): lua-5.4.4-4.el9.x86_64.rpm 2.8 MB/s | 188 kB 00:00 2026-03-20T11:20:53.175 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 66/140 2026-03-20T11:20:53.184 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 67/140 2026-03-20T11:20:53.222 INFO:teuthology.orchestra.run.vm08.stdout:(66/138): openblas-0.3.29-1.el9.x86_64.rpm 760 kB/s | 42 kB 00:00 2026-03-20T11:20:53.227 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 68/140 2026-03-20T11:20:53.244 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-protobuf-3.14.0-17.el9.noarch 69/140 2026-03-20T11:20:53.254 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 70/140 2026-03-20T11:20:53.266 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 71/140 2026-03-20T11:20:53.270 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-numpy-1:1.23.5-2.el9.x86_64 81/140 2026-03-20T11:20:53.316 INFO:teuthology.orchestra.run.vm08.stdout:(67/138): flexiblas-netlib-3.0.4-9.el9.x86_64.r 2.7 MB/s | 3.0 MB 00:01 2026-03-20T11:20:53.316 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-jmespath-1.0.1-1.el9.noarch 72/140 2026-03-20T11:20:53.373 INFO:teuthology.orchestra.run.vm08.stdout:(68/138): perl-Benchmark-1.23-483.el9.noarch.rp 463 kB/s | 26 kB 00:00 2026-03-20T11:20:53.569 INFO:teuthology.orchestra.run.vm08.stdout:(69/138): perl-Test-Harness-3.42-461.el9.noarch 1.5 MB/s | 295 kB 00:00 2026-03-20T11:20:53.655 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-devel-3.9.25-3.el9.x86_64 73/140 2026-03-20T11:20:53.694 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 74/140 2026-03-20T11:20:53.700 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-jinja2-2.11.3-8.el9.noarch 75/140 2026-03-20T11:20:53.707 INFO:teuthology.orchestra.run.vm05.stdout: Installing : perl-Benchmark-1.23-483.el9.noarch 76/140 2026-03-20T11:20:53.785 INFO:teuthology.orchestra.run.vm05.stdout: Installing : openblas-0.3.29-1.el9.x86_64 77/140 2026-03-20T11:20:53.789 INFO:teuthology.orchestra.run.vm05.stdout: Installing : openblas-openmp-0.3.29-1.el9.x86_64 78/140 2026-03-20T11:20:53.820 INFO:teuthology.orchestra.run.vm05.stdout: Installing : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 79/140 2026-03-20T11:20:54.164 INFO:teuthology.orchestra.run.vm08.stdout:(70/138): openblas-openmp-0.3.29-1.el9.x86_64.r 5.6 MB/s | 5.3 MB 00:00 2026-03-20T11:20:54.168 INFO:teuthology.orchestra.run.vm08.stdout:(71/138): protobuf-3.14.0-17.el9.x86_64.rpm 1.7 MB/s | 1.0 MB 00:00 2026-03-20T11:20:54.265 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 82/140 2026-03-20T11:20:54.297 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-scipy-1.9.3-2.el9.x86_64 83/140 2026-03-20T11:20:54.305 INFO:teuthology.orchestra.run.vm07.stdout: Installing : libxslt-1.1.34-12.el9.x86_64 84/140 2026-03-20T11:20:54.307 INFO:teuthology.orchestra.run.vm05.stdout: Installing : flexiblas-netlib-3.0.4-9.el9.x86_64 80/140 2026-03-20T11:20:54.310 INFO:teuthology.orchestra.run.vm07.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 85/140 2026-03-20T11:20:54.320 INFO:teuthology.orchestra.run.vm07.stdout: Installing : boost-program-options-1.75.0-13.el9.x86_64 86/140 2026-03-20T11:20:54.416 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-numpy-1:1.23.5-2.el9.x86_64 81/140 2026-03-20T11:20:54.689 INFO:teuthology.orchestra.run.vm07.stdout: Installing : parquet-libs-9.0.0-15.el9.x86_64 87/140 2026-03-20T11:20:54.692 INFO:teuthology.orchestra.run.vm07.stdout: Installing : librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 88/140 2026-03-20T11:20:54.723 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 88/140 2026-03-20T11:20:54.725 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-rgw-2:20.2.0-712.g70f8415b.el9.x86_64 89/140 2026-03-20T11:20:55.003 INFO:teuthology.orchestra.run.vm08.stdout:(72/138): python3-devel-3.9.25-3.el9.x86_64.rpm 293 kB/s | 244 kB 00:00 2026-03-20T11:20:55.243 INFO:teuthology.orchestra.run.vm08.stdout:(73/138): python3-jinja2-2.11.3-8.el9.noarch.rp 1.0 MB/s | 249 kB 00:00 2026-03-20T11:20:55.404 INFO:teuthology.orchestra.run.vm08.stdout:(74/138): python3-jmespath-1.0.1-1.el9.noarch.r 296 kB/s | 48 kB 00:00 2026-03-20T11:20:55.408 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 82/140 2026-03-20T11:20:55.437 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-scipy-1.9.3-2.el9.x86_64 83/140 2026-03-20T11:20:55.447 INFO:teuthology.orchestra.run.vm05.stdout: Installing : libxslt-1.1.34-12.el9.x86_64 84/140 2026-03-20T11:20:55.453 INFO:teuthology.orchestra.run.vm05.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 85/140 2026-03-20T11:20:55.461 INFO:teuthology.orchestra.run.vm05.stdout: Installing : boost-program-options-1.75.0-13.el9.x86_64 86/140 2026-03-20T11:20:55.591 INFO:teuthology.orchestra.run.vm08.stdout:(75/138): python3-libstoragemgmt-1.10.1-1.el9.x 951 kB/s | 177 kB 00:00 2026-03-20T11:20:55.670 INFO:teuthology.orchestra.run.vm08.stdout:(76/138): python3-markupsafe-1.1.1-12.el9.x86_6 441 kB/s | 35 kB 00:00 2026-03-20T11:20:55.826 INFO:teuthology.orchestra.run.vm05.stdout: Installing : parquet-libs-9.0.0-15.el9.x86_64 87/140 2026-03-20T11:20:55.829 INFO:teuthology.orchestra.run.vm05.stdout: Installing : librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 88/140 2026-03-20T11:20:55.860 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 88/140 2026-03-20T11:20:56.005 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-rgw-2:20.2.0-712.g70f8415b.el9.x86_64 89/140 2026-03-20T11:20:56.029 INFO:teuthology.orchestra.run.vm08.stdout:(77/138): python3-babel-2.9.1-2.el9.noarch.rpm 3.2 MB/s | 6.0 MB 00:01 2026-03-20T11:20:56.173 INFO:teuthology.orchestra.run.vm08.stdout:(78/138): python3-numpy-f2py-1.23.5-2.el9.x86_6 3.0 MB/s | 442 kB 00:00 2026-03-20T11:20:56.184 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 90/140 2026-03-20T11:20:56.295 INFO:teuthology.orchestra.run.vm08.stdout:(79/138): python3-packaging-20.9-5.el9.noarch.r 638 kB/s | 77 kB 00:00 2026-03-20T11:20:56.299 INFO:teuthology.orchestra.run.vm02.stdout:(138/138): ceph-test-20.2.0-712.g70f8415b.el9.x 3.7 MB/s | 84 MB 00:22 2026-03-20T11:20:56.300 INFO:teuthology.orchestra.run.vm07.stdout: Installing : ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 90/140 2026-03-20T11:20:56.302 INFO:teuthology.orchestra.run.vm02.stdout:-------------------------------------------------------------------------------- 2026-03-20T11:20:56.303 INFO:teuthology.orchestra.run.vm02.stdout:Total 9.2 MB/s | 267 MB 00:28 2026-03-20T11:20:56.333 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 90/140 2026-03-20T11:20:56.353 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-pyparsing-2.4.7-9.el9.noarch 91/140 2026-03-20T11:20:56.364 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-packaging-20.9-5.el9.noarch 92/140 2026-03-20T11:20:56.385 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-ply-3.11-14.el9.noarch 93/140 2026-03-20T11:20:56.398 INFO:teuthology.orchestra.run.vm08.stdout:(80/138): python3-protobuf-3.14.0-17.el9.noarch 2.5 MB/s | 267 kB 00:00 2026-03-20T11:20:56.410 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 94/140 2026-03-20T11:20:56.432 INFO:teuthology.orchestra.run.vm08.stdout:(81/138): python3-pyasn1-0.4.8-7.el9.noarch.rpm 4.6 MB/s | 157 kB 00:00 2026-03-20T11:20:56.496 INFO:teuthology.orchestra.run.vm08.stdout:(82/138): python3-pyasn1-modules-0.4.8-7.el9.no 4.3 MB/s | 277 kB 00:00 2026-03-20T11:20:56.524 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 95/140 2026-03-20T11:20:56.543 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-cryptography-36.0.1-5.el9.x86_64 96/140 2026-03-20T11:20:56.548 INFO:teuthology.orchestra.run.vm08.stdout:(83/138): python3-requests-oauthlib-1.3.0-12.el 1.0 MB/s | 54 kB 00:00 2026-03-20T11:20:56.588 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 97/140 2026-03-20T11:20:56.647 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-cheroot-10.0.1-4.el9.noarch 98/140 2026-03-20T11:20:56.725 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 99/140 2026-03-20T11:20:56.737 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 100/140 2026-03-20T11:20:56.746 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 101/140 2026-03-20T11:20:56.754 INFO:teuthology.orchestra.run.vm07.stdout: Installing : pciutils-3.7.0-7.el9.x86_64 102/140 2026-03-20T11:20:56.761 INFO:teuthology.orchestra.run.vm07.stdout: Installing : qatlib-25.08.0-2.el9.x86_64 103/140 2026-03-20T11:20:56.762 INFO:teuthology.orchestra.run.vm07.stdout: Installing : qatlib-service-25.08.0-2.el9.x86_64 104/140 2026-03-20T11:20:56.792 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: qatlib-service-25.08.0-2.el9.x86_64 104/140 2026-03-20T11:20:57.026 INFO:teuthology.orchestra.run.vm02.stdout:Running transaction check 2026-03-20T11:20:57.095 INFO:teuthology.orchestra.run.vm02.stdout:Transaction check succeeded. 2026-03-20T11:20:57.095 INFO:teuthology.orchestra.run.vm02.stdout:Running transaction test 2026-03-20T11:20:57.192 INFO:teuthology.orchestra.run.vm07.stdout: Installing : qatzip-libs-1.3.1-1.el9.x86_64 105/140 2026-03-20T11:20:57.199 INFO:teuthology.orchestra.run.vm07.stdout: Installing : ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 106/140 2026-03-20T11:20:57.254 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 106/140 2026-03-20T11:20:57.254 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2026-03-20T11:20:57.254 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2026-03-20T11:20:57.255 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:20:57.260 INFO:teuthology.orchestra.run.vm07.stdout: Installing : ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 107/140 2026-03-20T11:20:57.429 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 90/140 2026-03-20T11:20:57.482 INFO:teuthology.orchestra.run.vm05.stdout: Installing : ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 90/140 2026-03-20T11:20:57.509 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 90/140 2026-03-20T11:20:57.528 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-pyparsing-2.4.7-9.el9.noarch 91/140 2026-03-20T11:20:57.540 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-packaging-20.9-5.el9.noarch 92/140 2026-03-20T11:20:57.560 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-ply-3.11-14.el9.noarch 93/140 2026-03-20T11:20:57.589 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 94/140 2026-03-20T11:20:57.713 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 95/140 2026-03-20T11:20:57.726 INFO:teuthology.orchestra.run.vm08.stdout:(84/138): ceph-test-20.2.0-712.g70f8415b.el9.x8 5.0 MB/s | 84 MB 00:16 2026-03-20T11:20:57.737 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-cryptography-36.0.1-5.el9.x86_64 96/140 2026-03-20T11:20:57.792 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 97/140 2026-03-20T11:20:57.840 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-cheroot-10.0.1-4.el9.noarch 98/140 2026-03-20T11:20:57.905 INFO:teuthology.orchestra.run.vm08.stdout:(85/138): python3-toml-0.10.2-6.el9.noarch.rpm 232 kB/s | 42 kB 00:00 2026-03-20T11:20:57.919 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 99/140 2026-03-20T11:20:57.926 INFO:teuthology.orchestra.run.vm08.stdout:(86/138): python3-numpy-1.23.5-2.el9.x86_64.rpm 2.7 MB/s | 6.1 MB 00:02 2026-03-20T11:20:57.937 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 100/140 2026-03-20T11:20:57.944 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 101/140 2026-03-20T11:20:57.952 INFO:teuthology.orchestra.run.vm05.stdout: Installing : pciutils-3.7.0-7.el9.x86_64 102/140 2026-03-20T11:20:57.957 INFO:teuthology.orchestra.run.vm05.stdout: Installing : qatlib-25.08.0-2.el9.x86_64 103/140 2026-03-20T11:20:57.959 INFO:teuthology.orchestra.run.vm05.stdout: Installing : qatlib-service-25.08.0-2.el9.x86_64 104/140 2026-03-20T11:20:57.982 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: qatlib-service-25.08.0-2.el9.x86_64 104/140 2026-03-20T11:20:58.005 INFO:teuthology.orchestra.run.vm08.stdout:(87/138): qatlib-service-25.08.0-2.el9.x86_64.r 472 kB/s | 37 kB 00:00 2026-03-20T11:20:58.118 INFO:teuthology.orchestra.run.vm08.stdout:(88/138): qatlib-25.08.0-2.el9.x86_64.rpm 1.1 MB/s | 240 kB 00:00 2026-03-20T11:20:58.165 INFO:teuthology.orchestra.run.vm08.stdout:(89/138): qatzip-libs-1.3.1-1.el9.x86_64.rpm 414 kB/s | 66 kB 00:00 2026-03-20T11:20:58.191 INFO:teuthology.orchestra.run.vm02.stdout:Transaction test succeeded. 2026-03-20T11:20:58.191 INFO:teuthology.orchestra.run.vm02.stdout:Running transaction 2026-03-20T11:20:58.259 INFO:teuthology.orchestra.run.vm08.stdout:(90/138): xmlstarlet-1.6.1-20.el9.x86_64.rpm 681 kB/s | 64 kB 00:00 2026-03-20T11:20:58.330 INFO:teuthology.orchestra.run.vm08.stdout:(91/138): socat-1.7.4.1-8.el9.x86_64.rpm 1.4 MB/s | 303 kB 00:00 2026-03-20T11:20:58.367 INFO:teuthology.orchestra.run.vm05.stdout: Installing : qatzip-libs-1.3.1-1.el9.x86_64 105/140 2026-03-20T11:20:58.374 INFO:teuthology.orchestra.run.vm05.stdout: Installing : ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 106/140 2026-03-20T11:20:58.422 INFO:teuthology.orchestra.run.vm08.stdout:(92/138): lua-devel-5.4.4-4.el9.x86_64.rpm 137 kB/s | 22 kB 00:00 2026-03-20T11:20:58.431 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 106/140 2026-03-20T11:20:58.431 INFO:teuthology.orchestra.run.vm05.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2026-03-20T11:20:58.431 INFO:teuthology.orchestra.run.vm05.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2026-03-20T11:20:58.431 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:20:58.436 INFO:teuthology.orchestra.run.vm05.stdout: Installing : ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 107/140 2026-03-20T11:20:58.733 INFO:teuthology.orchestra.run.vm08.stdout:(93/138): protobuf-compiler-3.14.0-17.el9.x86_6 2.1 MB/s | 862 kB 00:00 2026-03-20T11:20:58.872 INFO:teuthology.orchestra.run.vm08.stdout:(94/138): abseil-cpp-20211102.0-4.el9.x86_64.rp 1.2 MB/s | 551 kB 00:00 2026-03-20T11:20:58.940 INFO:teuthology.orchestra.run.vm08.stdout:(95/138): grpc-data-1.46.7-10.el9.noarch.rpm 285 kB/s | 19 kB 00:00 2026-03-20T11:20:59.045 INFO:teuthology.orchestra.run.vm08.stdout:(96/138): gperftools-libs-2.9.1-3.el9.x86_64.rp 986 kB/s | 308 kB 00:00 2026-03-20T11:20:59.104 INFO:teuthology.orchestra.run.vm08.stdout:(97/138): libarrow-doc-9.0.0-15.el9.noarch.rpm 423 kB/s | 25 kB 00:00 2026-03-20T11:20:59.176 INFO:teuthology.orchestra.run.vm08.stdout:(98/138): liboath-2.6.12-1.el9.x86_64.rpm 691 kB/s | 49 kB 00:00 2026-03-20T11:20:59.235 INFO:teuthology.orchestra.run.vm08.stdout:(99/138): libunwind-1.6.2-1.el9.x86_64.rpm 1.1 MB/s | 67 kB 00:00 2026-03-20T11:20:59.265 INFO:teuthology.orchestra.run.vm08.stdout:(100/138): libarrow-9.0.0-15.el9.x86_64.rpm 14 MB/s | 4.4 MB 00:00 2026-03-20T11:20:59.300 INFO:teuthology.orchestra.run.vm08.stdout:(101/138): luarocks-3.9.2-5.el9.noarch.rpm 2.2 MB/s | 151 kB 00:00 2026-03-20T11:20:59.344 INFO:teuthology.orchestra.run.vm08.stdout:(102/138): parquet-libs-9.0.0-15.el9.x86_64.rpm 10 MB/s | 838 kB 00:00 2026-03-20T11:20:59.408 INFO:teuthology.orchestra.run.vm08.stdout:(103/138): python3-autocommand-2.2.2-8.el9.noar 462 kB/s | 29 kB 00:00 2026-03-20T11:20:59.415 INFO:teuthology.orchestra.run.vm02.stdout: Preparing : 1/1 2026-03-20T11:20:59.417 INFO:teuthology.orchestra.run.vm08.stdout:(104/138): python3-asyncssh-2.13.2-5.el9.noarch 4.6 MB/s | 548 kB 00:00 2026-03-20T11:20:59.426 INFO:teuthology.orchestra.run.vm02.stdout: Installing : thrift-0.15.0-4.el9.x86_64 1/140 2026-03-20T11:20:59.429 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 2/140 2026-03-20T11:20:59.444 INFO:teuthology.orchestra.run.vm02.stdout: Installing : liboath-2.6.12-1.el9.x86_64 3/140 2026-03-20T11:20:59.467 INFO:teuthology.orchestra.run.vm08.stdout:(105/138): python3-backports-tarfile-1.2.0-1.el 1.0 MB/s | 60 kB 00:00 2026-03-20T11:20:59.472 INFO:teuthology.orchestra.run.vm08.stdout:(106/138): python3-bcrypt-3.2.2-1.el9.x86_64.rp 788 kB/s | 43 kB 00:00 2026-03-20T11:20:59.525 INFO:teuthology.orchestra.run.vm08.stdout:(107/138): python3-certifi-2023.05.07-4.el9.noa 267 kB/s | 14 kB 00:00 2026-03-20T11:20:59.530 INFO:teuthology.orchestra.run.vm08.stdout:(108/138): python3-cachetools-4.2.4-1.el9.noarc 512 kB/s | 32 kB 00:00 2026-03-20T11:20:59.588 INFO:teuthology.orchestra.run.vm08.stdout:(109/138): python3-cheroot-10.0.1-4.el9.noarch. 2.7 MB/s | 173 kB 00:00 2026-03-20T11:20:59.599 INFO:teuthology.orchestra.run.vm08.stdout:(110/138): python3-cherrypy-18.6.1-2.el9.noarch 5.1 MB/s | 358 kB 00:00 2026-03-20T11:20:59.638 INFO:teuthology.orchestra.run.vm08.stdout:(111/138): python3-scipy-1.9.3-2.el9.x86_64.rpm 6.2 MB/s | 19 MB 00:03 2026-03-20T11:20:59.645 INFO:teuthology.orchestra.run.vm02.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 4/140 2026-03-20T11:20:59.649 INFO:teuthology.orchestra.run.vm02.stdout: Upgrading : librados2-2:20.2.0-712.g70f8415b.el9.x86_64 5/140 2026-03-20T11:20:59.656 INFO:teuthology.orchestra.run.vm08.stdout:(112/138): python3-google-auth-2.45.0-1.el9.noa 3.6 MB/s | 254 kB 00:00 2026-03-20T11:20:59.696 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: librados2-2:20.2.0-712.g70f8415b.el9.x86_64 5/140 2026-03-20T11:20:59.710 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-rados-2:20.2.0-712.g70f8415b.el9.x86_64 6/140 2026-03-20T11:20:59.714 INFO:teuthology.orchestra.run.vm02.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 7/140 2026-03-20T11:20:59.716 INFO:teuthology.orchestra.run.vm08.stdout:(113/138): python3-jaraco-8.2.1-3.el9.noarch.rp 179 kB/s | 11 kB 00:00 2026-03-20T11:20:59.719 INFO:teuthology.orchestra.run.vm02.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 8/140 2026-03-20T11:20:59.723 INFO:teuthology.orchestra.run.vm02.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 9/140 2026-03-20T11:20:59.729 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 10/140 2026-03-20T11:20:59.775 INFO:teuthology.orchestra.run.vm08.stdout:(114/138): python3-jaraco-classes-3.2.1-5.el9.n 305 kB/s | 18 kB 00:00 2026-03-20T11:20:59.840 INFO:teuthology.orchestra.run.vm08.stdout:(115/138): python3-jaraco-collections-3.0.0-8.e 358 kB/s | 23 kB 00:00 2026-03-20T11:20:59.897 INFO:teuthology.orchestra.run.vm08.stdout:(116/138): python3-grpcio-tools-1.46.7-10.el9.x 558 kB/s | 144 kB 00:00 2026-03-20T11:20:59.904 INFO:teuthology.orchestra.run.vm02.stdout: Installing : libnbd-1.20.3-4.el9.x86_64 11/140 2026-03-20T11:20:59.906 INFO:teuthology.orchestra.run.vm02.stdout: Upgrading : librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 12/140 2026-03-20T11:20:59.908 INFO:teuthology.orchestra.run.vm08.stdout:(117/138): python3-jaraco-context-6.0.1-3.el9.n 288 kB/s | 20 kB 00:00 2026-03-20T11:20:59.931 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 12/140 2026-03-20T11:20:59.932 INFO:teuthology.orchestra.run.vm02.stdout: Installing : libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 13/140 2026-03-20T11:20:59.949 INFO:teuthology.orchestra.run.vm08.stdout:(118/138): python3-jaraco-functools-3.5.0-2.el9 379 kB/s | 19 kB 00:00 2026-03-20T11:20:59.961 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 13/140 2026-03-20T11:20:59.963 INFO:teuthology.orchestra.run.vm02.stdout: Installing : libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_ 14/140 2026-03-20T11:20:59.971 INFO:teuthology.orchestra.run.vm08.stdout:(119/138): python3-jaraco-text-4.0.0-2.el9.noar 429 kB/s | 26 kB 00:00 2026-03-20T11:20:59.984 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_ 14/140 2026-03-20T11:21:00.024 INFO:teuthology.orchestra.run.vm02.stdout: Installing : re2-1:20211101-20.el9.x86_64 15/140 2026-03-20T11:21:00.027 INFO:teuthology.orchestra.run.vm08.stdout:(120/138): python3-more-itertools-8.12.0-2.el9. 1.4 MB/s | 79 kB 00:00 2026-03-20T11:21:00.059 INFO:teuthology.orchestra.run.vm02.stdout: Installing : libarrow-9.0.0-15.el9.x86_64 16/140 2026-03-20T11:21:00.074 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-pyasn1-0.4.8-7.el9.noarch 17/140 2026-03-20T11:21:00.083 INFO:teuthology.orchestra.run.vm02.stdout: Installing : protobuf-3.14.0-17.el9.x86_64 18/140 2026-03-20T11:21:00.087 INFO:teuthology.orchestra.run.vm02.stdout: Installing : lua-5.4.4-4.el9.x86_64 19/140 2026-03-20T11:21:00.091 INFO:teuthology.orchestra.run.vm08.stdout:(121/138): python3-natsort-7.1.1-5.el9.noarch.r 916 kB/s | 58 kB 00:00 2026-03-20T11:21:00.096 INFO:teuthology.orchestra.run.vm02.stdout: Installing : flexiblas-3.0.4-9.el9.x86_64 20/140 2026-03-20T11:21:00.099 INFO:teuthology.orchestra.run.vm08.stdout:(122/138): python3-grpcio-1.46.7-10.el9.x86_64. 4.1 MB/s | 2.0 MB 00:00 2026-03-20T11:21:00.126 INFO:teuthology.orchestra.run.vm02.stdout: Installing : unzip-6.0-59.el9.x86_64 21/140 2026-03-20T11:21:00.147 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-urllib3-1.26.5-7.el9.noarch 22/140 2026-03-20T11:21:00.148 INFO:teuthology.orchestra.run.vm08.stdout:(123/138): python3-portend-3.1.0-2.el9.noarch.r 286 kB/s | 16 kB 00:00 2026-03-20T11:21:00.153 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-requests-2.25.1-10.el9.noarch 23/140 2026-03-20T11:21:00.159 INFO:teuthology.orchestra.run.vm08.stdout:(124/138): python3-pyOpenSSL-21.0.0-1.el9.noarc 1.5 MB/s | 90 kB 00:00 2026-03-20T11:21:00.166 INFO:teuthology.orchestra.run.vm02.stdout: Installing : libquadmath-11.5.0-14.el9.x86_64 24/140 2026-03-20T11:21:00.170 INFO:teuthology.orchestra.run.vm02.stdout: Installing : libgfortran-11.5.0-14.el9.x86_64 25/140 2026-03-20T11:21:00.173 INFO:teuthology.orchestra.run.vm08.stdout:(125/138): python3-kubernetes-26.1.0-3.el9.noar 4.6 MB/s | 1.0 MB 00:00 2026-03-20T11:21:00.215 INFO:teuthology.orchestra.run.vm08.stdout:(126/138): python3-repoze-lru-0.7-16.el9.noarch 468 kB/s | 31 kB 00:00 2026-03-20T11:21:00.217 INFO:teuthology.orchestra.run.vm02.stdout: Installing : ledmon-libs-1.1.0-3.el9.x86_64 26/140 2026-03-20T11:21:00.220 INFO:teuthology.orchestra.run.vm08.stdout:(127/138): python3-routes-2.5.1-5.el9.noarch.rp 3.0 MB/s | 188 kB 00:00 2026-03-20T11:21:00.225 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-ceph-common-2:20.2.0-712.g70f8415b.el9.x 27/140 2026-03-20T11:21:00.227 INFO:teuthology.orchestra.run.vm08.stdout:(128/138): python3-rsa-4.9-2.el9.noarch.rpm 1.1 MB/s | 59 kB 00:00 2026-03-20T11:21:00.229 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-ceph-argparse-2:20.2.0-712.g70f8415b.el9 28/140 2026-03-20T11:21:00.230 INFO:teuthology.orchestra.run.vm02.stdout: Installing : libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_ 29/140 2026-03-20T11:21:00.287 INFO:teuthology.orchestra.run.vm08.stdout:(129/138): python3-typing-extensions-4.15.0-1.e 1.3 MB/s | 86 kB 00:00 2026-03-20T11:21:00.288 INFO:teuthology.orchestra.run.vm08.stdout:(130/138): python3-tempora-5.0.0-2.el9.noarch.r 485 kB/s | 36 kB 00:00 2026-03-20T11:21:00.291 INFO:teuthology.orchestra.run.vm08.stdout:(131/138): python3-websocket-client-1.2.3-2.el9 1.4 MB/s | 90 kB 00:00 2026-03-20T11:21:00.300 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_ 29/140 2026-03-20T11:21:00.302 INFO:teuthology.orchestra.run.vm02.stdout: Installing : libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 30/140 2026-03-20T11:21:00.332 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 30/140 2026-03-20T11:21:00.348 INFO:teuthology.orchestra.run.vm08.stdout:(132/138): python3-xmltodict-0.12.0-15.el9.noar 366 kB/s | 22 kB 00:00 2026-03-20T11:21:00.351 INFO:teuthology.orchestra.run.vm08.stdout:(133/138): python3-zc-lockfile-2.0-10.el9.noarc 318 kB/s | 20 kB 00:00 2026-03-20T11:21:00.353 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-cephfs-2:20.2.0-712.g70f8415b.el9.x86_64 31/140 2026-03-20T11:21:00.356 INFO:teuthology.orchestra.run.vm08.stdout:(134/138): re2-20211101-20.el9.x86_64.rpm 2.9 MB/s | 191 kB 00:00 2026-03-20T11:21:00.365 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 32/140 2026-03-20T11:21:00.399 INFO:teuthology.orchestra.run.vm02.stdout: Installing : zip-3.0-35.el9.x86_64 33/140 2026-03-20T11:21:00.405 INFO:teuthology.orchestra.run.vm02.stdout: Installing : luarocks-3.9.2-5.el9.noarch 34/140 2026-03-20T11:21:00.414 INFO:teuthology.orchestra.run.vm08.stdout:(135/138): s3cmd-2.4.0-1.el9.noarch.rpm 3.1 MB/s | 206 kB 00:00 2026-03-20T11:21:00.415 INFO:teuthology.orchestra.run.vm02.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 35/140 2026-03-20T11:21:00.489 INFO:teuthology.orchestra.run.vm08.stdout:(136/138): thrift-0.15.0-4.el9.x86_64.rpm 12 MB/s | 1.6 MB 00:00 2026-03-20T11:21:00.490 INFO:teuthology.orchestra.run.vm02.stdout: Installing : protobuf-compiler-3.14.0-17.el9.x86_64 36/140 2026-03-20T11:21:00.508 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-pyasn1-modules-0.4.8-7.el9.noarch 37/140 2026-03-20T11:21:00.530 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-rsa-4.9-2.el9.noarch 38/140 2026-03-20T11:21:00.539 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-rbd-2:20.2.0-712.g70f8415b.el9.x86_64 39/140 2026-03-20T11:21:00.552 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 40/140 2026-03-20T11:21:00.562 INFO:teuthology.orchestra.run.vm02.stdout: Installing : librados-devel-2:20.2.0-712.g70f8415b.el9.x86_64 41/140 2026-03-20T11:21:00.568 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 42/140 2026-03-20T11:21:00.598 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-xmltodict-0.12.0-15.el9.noarch 43/140 2026-03-20T11:21:00.605 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 44/140 2026-03-20T11:21:00.612 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-typing-extensions-4.15.0-1.el9.noarch 45/140 2026-03-20T11:21:00.628 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 46/140 2026-03-20T11:21:00.641 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 47/140 2026-03-20T11:21:00.726 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 48/140 2026-03-20T11:21:00.744 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 49/140 2026-03-20T11:21:00.796 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 50/140 2026-03-20T11:21:01.226 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-google-auth-1:2.45.0-1.el9.noarch 51/140 2026-03-20T11:21:01.242 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-kubernetes-1:26.1.0-3.el9.noarch 52/140 2026-03-20T11:21:01.248 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-backports-tarfile-1.2.0-1.el9.noarch 53/140 2026-03-20T11:21:01.257 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-jaraco-context-6.0.1-3.el9.noarch 54/140 2026-03-20T11:21:01.261 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-autocommand-2.2.2-8.el9.noarch 55/140 2026-03-20T11:21:01.269 INFO:teuthology.orchestra.run.vm02.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 56/140 2026-03-20T11:21:01.272 INFO:teuthology.orchestra.run.vm02.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 57/140 2026-03-20T11:21:01.275 INFO:teuthology.orchestra.run.vm02.stdout: Installing : libarrow-doc-9.0.0-15.el9.noarch 58/140 2026-03-20T11:21:01.318 INFO:teuthology.orchestra.run.vm02.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 59/140 2026-03-20T11:21:01.380 INFO:teuthology.orchestra.run.vm02.stdout: Installing : abseil-cpp-20211102.0-4.el9.x86_64 60/140 2026-03-20T11:21:01.398 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 61/140 2026-03-20T11:21:01.408 INFO:teuthology.orchestra.run.vm02.stdout: Installing : socat-1.7.4.1-8.el9.x86_64 62/140 2026-03-20T11:21:01.413 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 63/140 2026-03-20T11:21:01.422 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 64/140 2026-03-20T11:21:01.427 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-jaraco-text-4.0.0-2.el9.noarch 65/140 2026-03-20T11:21:01.438 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 66/140 2026-03-20T11:21:01.443 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 67/140 2026-03-20T11:21:01.480 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 68/140 2026-03-20T11:21:01.494 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-protobuf-3.14.0-17.el9.noarch 69/140 2026-03-20T11:21:01.504 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 70/140 2026-03-20T11:21:01.514 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 71/140 2026-03-20T11:21:01.560 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-jmespath-1.0.1-1.el9.noarch 72/140 2026-03-20T11:21:01.610 INFO:teuthology.orchestra.run.vm08.stdout:(137/138): librbd1-20.2.0-712.g70f8415b.el9.x86 2.4 MB/s | 2.8 MB 00:01 2026-03-20T11:21:01.893 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-devel-3.9.25-3.el9.x86_64 73/140 2026-03-20T11:21:01.962 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 74/140 2026-03-20T11:21:01.996 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-jinja2-2.11.3-8.el9.noarch 75/140 2026-03-20T11:21:02.009 INFO:teuthology.orchestra.run.vm02.stdout: Installing : perl-Benchmark-1.23-483.el9.noarch 76/140 2026-03-20T11:21:02.100 INFO:teuthology.orchestra.run.vm02.stdout: Installing : openblas-0.3.29-1.el9.x86_64 77/140 2026-03-20T11:21:02.103 INFO:teuthology.orchestra.run.vm02.stdout: Installing : openblas-openmp-0.3.29-1.el9.x86_64 78/140 2026-03-20T11:21:02.132 INFO:teuthology.orchestra.run.vm02.stdout: Installing : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 79/140 2026-03-20T11:21:02.560 INFO:teuthology.orchestra.run.vm02.stdout: Installing : flexiblas-netlib-3.0.4-9.el9.x86_64 80/140 2026-03-20T11:21:02.661 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-numpy-1:1.23.5-2.el9.x86_64 81/140 2026-03-20T11:21:02.846 INFO:teuthology.orchestra.run.vm08.stdout:(138/138): librados2-20.2.0-712.g70f8415b.el9.x 1.4 MB/s | 3.5 MB 00:02 2026-03-20T11:21:02.849 INFO:teuthology.orchestra.run.vm08.stdout:-------------------------------------------------------------------------------- 2026-03-20T11:21:02.849 INFO:teuthology.orchestra.run.vm08.stdout:Total 9.0 MB/s | 267 MB 00:29 2026-03-20T11:21:03.494 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction check 2026-03-20T11:21:03.558 INFO:teuthology.orchestra.run.vm08.stdout:Transaction check succeeded. 2026-03-20T11:21:03.558 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction test 2026-03-20T11:21:03.596 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 82/140 2026-03-20T11:21:03.650 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-scipy-1.9.3-2.el9.x86_64 83/140 2026-03-20T11:21:03.734 INFO:teuthology.orchestra.run.vm02.stdout: Installing : libxslt-1.1.34-12.el9.x86_64 84/140 2026-03-20T11:21:03.748 INFO:teuthology.orchestra.run.vm02.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 85/140 2026-03-20T11:21:03.854 INFO:teuthology.orchestra.run.vm02.stdout: Installing : boost-program-options-1.75.0-13.el9.x86_64 86/140 2026-03-20T11:21:04.217 INFO:teuthology.orchestra.run.vm02.stdout: Installing : parquet-libs-9.0.0-15.el9.x86_64 87/140 2026-03-20T11:21:04.220 INFO:teuthology.orchestra.run.vm02.stdout: Installing : librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 88/140 2026-03-20T11:21:04.247 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 88/140 2026-03-20T11:21:04.249 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-rgw-2:20.2.0-712.g70f8415b.el9.x86_64 89/140 2026-03-20T11:21:04.540 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 107/140 2026-03-20T11:21:04.540 INFO:teuthology.orchestra.run.vm07.stdout:skipping the directory /sys 2026-03-20T11:21:04.540 INFO:teuthology.orchestra.run.vm07.stdout:skipping the directory /proc 2026-03-20T11:21:04.541 INFO:teuthology.orchestra.run.vm07.stdout:skipping the directory /mnt 2026-03-20T11:21:04.541 INFO:teuthology.orchestra.run.vm07.stdout:skipping the directory /var/tmp 2026-03-20T11:21:04.541 INFO:teuthology.orchestra.run.vm07.stdout:skipping the directory /home 2026-03-20T11:21:04.541 INFO:teuthology.orchestra.run.vm07.stdout:skipping the directory /root 2026-03-20T11:21:04.541 INFO:teuthology.orchestra.run.vm07.stdout:skipping the directory /tmp 2026-03-20T11:21:04.541 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:04.656 INFO:teuthology.orchestra.run.vm08.stdout:Transaction test succeeded. 2026-03-20T11:21:04.656 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction 2026-03-20T11:21:04.742 INFO:teuthology.orchestra.run.vm07.stdout: Installing : ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 108/140 2026-03-20T11:21:04.771 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 108/140 2026-03-20T11:21:04.771 INFO:teuthology.orchestra.run.vm07.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:04.771 INFO:teuthology.orchestra.run.vm07.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2026-03-20T11:21:04.771 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-03-20T11:21:04.771 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-03-20T11:21:04.771 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:05.038 INFO:teuthology.orchestra.run.vm07.stdout: Installing : ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 109/140 2026-03-20T11:21:05.067 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 109/140 2026-03-20T11:21:05.067 INFO:teuthology.orchestra.run.vm07.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:05.067 INFO:teuthology.orchestra.run.vm07.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2026-03-20T11:21:05.067 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-03-20T11:21:05.067 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-03-20T11:21:05.067 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:05.079 INFO:teuthology.orchestra.run.vm07.stdout: Installing : mailcap-2.1.49-5.el9.noarch 110/140 2026-03-20T11:21:05.082 INFO:teuthology.orchestra.run.vm07.stdout: Installing : libconfig-1.7.2-9.el9.x86_64 111/140 2026-03-20T11:21:05.105 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 112/140 2026-03-20T11:21:05.105 INFO:teuthology.orchestra.run.vm07.stdout:Creating group 'qat' with GID 994. 2026-03-20T11:21:05.105 INFO:teuthology.orchestra.run.vm07.stdout:Creating group 'libstoragemgmt' with GID 993. 2026-03-20T11:21:05.106 INFO:teuthology.orchestra.run.vm07.stdout:Creating user 'libstoragemgmt' (daemon account for libstoragemgmt) with UID 993 and GID 993. 2026-03-20T11:21:05.106 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:05.119 INFO:teuthology.orchestra.run.vm07.stdout: Installing : libstoragemgmt-1.10.1-1.el9.x86_64 112/140 2026-03-20T11:21:05.158 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 112/140 2026-03-20T11:21:05.158 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/libstoragemgmt.service → /usr/lib/systemd/system/libstoragemgmt.service. 2026-03-20T11:21:05.158 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:05.185 INFO:teuthology.orchestra.run.vm07.stdout: Installing : python3-libstoragemgmt-1.10.1-1.el9.x86_64 113/140 2026-03-20T11:21:05.225 INFO:teuthology.orchestra.run.vm07.stdout: Installing : fuse-2.9.9-17.el9.x86_64 114/140 2026-03-20T11:21:05.316 INFO:teuthology.orchestra.run.vm07.stdout: Installing : cryptsetup-2.8.1-3.el9.x86_64 115/140 2026-03-20T11:21:05.321 INFO:teuthology.orchestra.run.vm07.stdout: Installing : ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 116/140 2026-03-20T11:21:05.341 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 116/140 2026-03-20T11:21:05.341 INFO:teuthology.orchestra.run.vm07.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:05.341 INFO:teuthology.orchestra.run.vm07.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2026-03-20T11:21:05.341 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:05.526 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 107/140 2026-03-20T11:21:05.526 INFO:teuthology.orchestra.run.vm05.stdout:skipping the directory /sys 2026-03-20T11:21:05.526 INFO:teuthology.orchestra.run.vm05.stdout:skipping the directory /proc 2026-03-20T11:21:05.526 INFO:teuthology.orchestra.run.vm05.stdout:skipping the directory /mnt 2026-03-20T11:21:05.526 INFO:teuthology.orchestra.run.vm05.stdout:skipping the directory /var/tmp 2026-03-20T11:21:05.526 INFO:teuthology.orchestra.run.vm05.stdout:skipping the directory /home 2026-03-20T11:21:05.526 INFO:teuthology.orchestra.run.vm05.stdout:skipping the directory /root 2026-03-20T11:21:05.526 INFO:teuthology.orchestra.run.vm05.stdout:skipping the directory /tmp 2026-03-20T11:21:05.526 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:05.641 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 90/140 2026-03-20T11:21:05.666 INFO:teuthology.orchestra.run.vm02.stdout: Installing : ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 90/140 2026-03-20T11:21:05.667 INFO:teuthology.orchestra.run.vm05.stdout: Installing : ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 108/140 2026-03-20T11:21:05.689 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 90/140 2026-03-20T11:21:05.702 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-pyparsing-2.4.7-9.el9.noarch 91/140 2026-03-20T11:21:05.704 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 108/140 2026-03-20T11:21:05.704 INFO:teuthology.orchestra.run.vm05.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:05.704 INFO:teuthology.orchestra.run.vm05.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2026-03-20T11:21:05.705 INFO:teuthology.orchestra.run.vm05.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-03-20T11:21:05.705 INFO:teuthology.orchestra.run.vm05.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-03-20T11:21:05.705 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:05.712 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-packaging-20.9-5.el9.noarch 92/140 2026-03-20T11:21:05.739 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-ply-3.11-14.el9.noarch 93/140 2026-03-20T11:21:05.762 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 94/140 2026-03-20T11:21:05.856 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 95/140 2026-03-20T11:21:05.871 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-cryptography-36.0.1-5.el9.x86_64 96/140 2026-03-20T11:21:05.904 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 97/140 2026-03-20T11:21:05.945 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-cheroot-10.0.1-4.el9.noarch 98/140 2026-03-20T11:21:05.980 INFO:teuthology.orchestra.run.vm05.stdout: Installing : ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 109/140 2026-03-20T11:21:06.012 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 99/140 2026-03-20T11:21:06.013 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 109/140 2026-03-20T11:21:06.013 INFO:teuthology.orchestra.run.vm05.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:06.013 INFO:teuthology.orchestra.run.vm05.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2026-03-20T11:21:06.013 INFO:teuthology.orchestra.run.vm05.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-03-20T11:21:06.013 INFO:teuthology.orchestra.run.vm05.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-03-20T11:21:06.013 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:06.024 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 100/140 2026-03-20T11:21:06.025 INFO:teuthology.orchestra.run.vm05.stdout: Installing : mailcap-2.1.49-5.el9.noarch 110/140 2026-03-20T11:21:06.029 INFO:teuthology.orchestra.run.vm05.stdout: Installing : libconfig-1.7.2-9.el9.x86_64 111/140 2026-03-20T11:21:06.033 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 101/140 2026-03-20T11:21:06.041 INFO:teuthology.orchestra.run.vm02.stdout: Installing : pciutils-3.7.0-7.el9.x86_64 102/140 2026-03-20T11:21:06.045 INFO:teuthology.orchestra.run.vm02.stdout: Installing : qatlib-25.08.0-2.el9.x86_64 103/140 2026-03-20T11:21:06.048 INFO:teuthology.orchestra.run.vm02.stdout: Installing : qatlib-service-25.08.0-2.el9.x86_64 104/140 2026-03-20T11:21:06.060 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 112/140 2026-03-20T11:21:06.060 INFO:teuthology.orchestra.run.vm05.stdout:Creating group 'qat' with GID 994. 2026-03-20T11:21:06.060 INFO:teuthology.orchestra.run.vm05.stdout:Creating group 'libstoragemgmt' with GID 993. 2026-03-20T11:21:06.060 INFO:teuthology.orchestra.run.vm05.stdout:Creating user 'libstoragemgmt' (daemon account for libstoragemgmt) with UID 993 and GID 993. 2026-03-20T11:21:06.060 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:06.073 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: qatlib-service-25.08.0-2.el9.x86_64 104/140 2026-03-20T11:21:06.076 INFO:teuthology.orchestra.run.vm05.stdout: Installing : libstoragemgmt-1.10.1-1.el9.x86_64 112/140 2026-03-20T11:21:06.113 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 112/140 2026-03-20T11:21:06.113 INFO:teuthology.orchestra.run.vm05.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/libstoragemgmt.service → /usr/lib/systemd/system/libstoragemgmt.service. 2026-03-20T11:21:06.113 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:06.129 INFO:teuthology.orchestra.run.vm08.stdout: Preparing : 1/1 2026-03-20T11:21:06.137 INFO:teuthology.orchestra.run.vm05.stdout: Installing : python3-libstoragemgmt-1.10.1-1.el9.x86_64 113/140 2026-03-20T11:21:06.138 INFO:teuthology.orchestra.run.vm08.stdout: Installing : thrift-0.15.0-4.el9.x86_64 1/140 2026-03-20T11:21:06.142 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 2/140 2026-03-20T11:21:06.155 INFO:teuthology.orchestra.run.vm08.stdout: Installing : liboath-2.6.12-1.el9.x86_64 3/140 2026-03-20T11:21:06.180 INFO:teuthology.orchestra.run.vm05.stdout: Installing : fuse-2.9.9-17.el9.x86_64 114/140 2026-03-20T11:21:06.203 INFO:teuthology.orchestra.run.vm07.stdout: Installing : ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 117/140 2026-03-20T11:21:06.238 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 117/140 2026-03-20T11:21:06.238 INFO:teuthology.orchestra.run.vm07.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:06.238 INFO:teuthology.orchestra.run.vm07.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2026-03-20T11:21:06.238 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-03-20T11:21:06.238 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-03-20T11:21:06.238 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:06.274 INFO:teuthology.orchestra.run.vm05.stdout: Installing : cryptsetup-2.8.1-3.el9.x86_64 115/140 2026-03-20T11:21:06.279 INFO:teuthology.orchestra.run.vm05.stdout: Installing : ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 116/140 2026-03-20T11:21:06.299 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 116/140 2026-03-20T11:21:06.299 INFO:teuthology.orchestra.run.vm05.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:06.299 INFO:teuthology.orchestra.run.vm05.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2026-03-20T11:21:06.299 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:06.323 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: cephadm-2:20.2.0-712.g70f8415b.el9.noarch 118/140 2026-03-20T11:21:06.327 INFO:teuthology.orchestra.run.vm07.stdout: Installing : cephadm-2:20.2.0-712.g70f8415b.el9.noarch 118/140 2026-03-20T11:21:06.335 INFO:teuthology.orchestra.run.vm08.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 4/140 2026-03-20T11:21:06.337 INFO:teuthology.orchestra.run.vm08.stdout: Upgrading : librados2-2:20.2.0-712.g70f8415b.el9.x86_64 5/140 2026-03-20T11:21:06.337 INFO:teuthology.orchestra.run.vm07.stdout: Installing : ceph-prometheus-alerts-2:20.2.0-712.g70f8415b.el 119/140 2026-03-20T11:21:06.370 INFO:teuthology.orchestra.run.vm07.stdout: Installing : ceph-grafana-dashboards-2:20.2.0-712.g70f8415b.e 120/140 2026-03-20T11:21:06.373 INFO:teuthology.orchestra.run.vm07.stdout: Installing : ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noar 121/140 2026-03-20T11:21:06.374 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: librados2-2:20.2.0-712.g70f8415b.el9.x86_64 5/140 2026-03-20T11:21:06.384 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-rados-2:20.2.0-712.g70f8415b.el9.x86_64 6/140 2026-03-20T11:21:06.388 INFO:teuthology.orchestra.run.vm08.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 7/140 2026-03-20T11:21:06.392 INFO:teuthology.orchestra.run.vm08.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 8/140 2026-03-20T11:21:06.394 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 9/140 2026-03-20T11:21:06.402 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 10/140 2026-03-20T11:21:06.443 INFO:teuthology.orchestra.run.vm02.stdout: Installing : qatzip-libs-1.3.1-1.el9.x86_64 105/140 2026-03-20T11:21:06.460 INFO:teuthology.orchestra.run.vm02.stdout: Installing : ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 106/140 2026-03-20T11:21:06.515 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 106/140 2026-03-20T11:21:06.516 INFO:teuthology.orchestra.run.vm02.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2026-03-20T11:21:06.516 INFO:teuthology.orchestra.run.vm02.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2026-03-20T11:21:06.516 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:06.523 INFO:teuthology.orchestra.run.vm02.stdout: Installing : ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 107/140 2026-03-20T11:21:06.611 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libnbd-1.20.3-4.el9.x86_64 11/140 2026-03-20T11:21:06.631 INFO:teuthology.orchestra.run.vm08.stdout: Upgrading : librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 12/140 2026-03-20T11:21:06.652 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 12/140 2026-03-20T11:21:06.654 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 13/140 2026-03-20T11:21:06.685 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 13/140 2026-03-20T11:21:06.687 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_ 14/140 2026-03-20T11:21:06.708 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_ 14/140 2026-03-20T11:21:06.748 INFO:teuthology.orchestra.run.vm08.stdout: Installing : re2-1:20211101-20.el9.x86_64 15/140 2026-03-20T11:21:06.779 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libarrow-9.0.0-15.el9.x86_64 16/140 2026-03-20T11:21:06.792 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-pyasn1-0.4.8-7.el9.noarch 17/140 2026-03-20T11:21:06.803 INFO:teuthology.orchestra.run.vm08.stdout: Installing : protobuf-3.14.0-17.el9.x86_64 18/140 2026-03-20T11:21:06.810 INFO:teuthology.orchestra.run.vm08.stdout: Installing : lua-5.4.4-4.el9.x86_64 19/140 2026-03-20T11:21:06.820 INFO:teuthology.orchestra.run.vm08.stdout: Installing : flexiblas-3.0.4-9.el9.x86_64 20/140 2026-03-20T11:21:06.851 INFO:teuthology.orchestra.run.vm08.stdout: Installing : unzip-6.0-59.el9.x86_64 21/140 2026-03-20T11:21:06.875 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-urllib3-1.26.5-7.el9.noarch 22/140 2026-03-20T11:21:06.880 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-requests-2.25.1-10.el9.noarch 23/140 2026-03-20T11:21:06.888 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libquadmath-11.5.0-14.el9.x86_64 24/140 2026-03-20T11:21:06.891 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libgfortran-11.5.0-14.el9.x86_64 25/140 2026-03-20T11:21:06.936 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ledmon-libs-1.1.0-3.el9.x86_64 26/140 2026-03-20T11:21:06.943 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-ceph-common-2:20.2.0-712.g70f8415b.el9.x 27/140 2026-03-20T11:21:06.946 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-ceph-argparse-2:20.2.0-712.g70f8415b.el9 28/140 2026-03-20T11:21:06.947 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_ 29/140 2026-03-20T11:21:07.010 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_ 29/140 2026-03-20T11:21:07.012 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 30/140 2026-03-20T11:21:07.036 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 30/140 2026-03-20T11:21:07.051 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-cephfs-2:20.2.0-712.g70f8415b.el9.x86_64 31/140 2026-03-20T11:21:07.060 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 32/140 2026-03-20T11:21:07.092 INFO:teuthology.orchestra.run.vm08.stdout: Installing : zip-3.0-35.el9.x86_64 33/140 2026-03-20T11:21:07.099 INFO:teuthology.orchestra.run.vm08.stdout: Installing : luarocks-3.9.2-5.el9.noarch 34/140 2026-03-20T11:21:07.109 INFO:teuthology.orchestra.run.vm08.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 35/140 2026-03-20T11:21:07.173 INFO:teuthology.orchestra.run.vm05.stdout: Installing : ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 117/140 2026-03-20T11:21:07.180 INFO:teuthology.orchestra.run.vm08.stdout: Installing : protobuf-compiler-3.14.0-17.el9.x86_64 36/140 2026-03-20T11:21:07.202 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 117/140 2026-03-20T11:21:07.202 INFO:teuthology.orchestra.run.vm05.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:07.202 INFO:teuthology.orchestra.run.vm05.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2026-03-20T11:21:07.202 INFO:teuthology.orchestra.run.vm05.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-03-20T11:21:07.202 INFO:teuthology.orchestra.run.vm05.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-03-20T11:21:07.202 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:07.207 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-pyasn1-modules-0.4.8-7.el9.noarch 37/140 2026-03-20T11:21:07.228 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-rsa-4.9-2.el9.noarch 38/140 2026-03-20T11:21:07.235 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-rbd-2:20.2.0-712.g70f8415b.el9.x86_64 39/140 2026-03-20T11:21:07.247 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 40/140 2026-03-20T11:21:07.255 INFO:teuthology.orchestra.run.vm08.stdout: Installing : librados-devel-2:20.2.0-712.g70f8415b.el9.x86_64 41/140 2026-03-20T11:21:07.259 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 42/140 2026-03-20T11:21:07.282 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-xmltodict-0.12.0-15.el9.noarch 43/140 2026-03-20T11:21:07.289 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 44/140 2026-03-20T11:21:07.294 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: cephadm-2:20.2.0-712.g70f8415b.el9.noarch 118/140 2026-03-20T11:21:07.297 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-typing-extensions-4.15.0-1.el9.noarch 45/140 2026-03-20T11:21:07.298 INFO:teuthology.orchestra.run.vm05.stdout: Installing : cephadm-2:20.2.0-712.g70f8415b.el9.noarch 118/140 2026-03-20T11:21:07.308 INFO:teuthology.orchestra.run.vm05.stdout: Installing : ceph-prometheus-alerts-2:20.2.0-712.g70f8415b.el 119/140 2026-03-20T11:21:07.311 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 46/140 2026-03-20T11:21:07.324 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 47/140 2026-03-20T11:21:07.330 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 48/140 2026-03-20T11:21:07.339 INFO:teuthology.orchestra.run.vm05.stdout: Installing : ceph-grafana-dashboards-2:20.2.0-712.g70f8415b.e 120/140 2026-03-20T11:21:07.340 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 49/140 2026-03-20T11:21:07.343 INFO:teuthology.orchestra.run.vm05.stdout: Installing : ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noar 121/140 2026-03-20T11:21:07.393 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 50/140 2026-03-20T11:21:07.783 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noar 121/140 2026-03-20T11:21:07.795 INFO:teuthology.orchestra.run.vm07.stdout: Installing : ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.no 122/140 2026-03-20T11:21:07.829 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-google-auth-1:2.45.0-1.el9.noarch 51/140 2026-03-20T11:21:07.844 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-kubernetes-1:26.1.0-3.el9.noarch 52/140 2026-03-20T11:21:07.850 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-backports-tarfile-1.2.0-1.el9.noarch 53/140 2026-03-20T11:21:07.858 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-jaraco-context-6.0.1-3.el9.noarch 54/140 2026-03-20T11:21:07.863 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-autocommand-2.2.2-8.el9.noarch 55/140 2026-03-20T11:21:07.870 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 56/140 2026-03-20T11:21:07.874 INFO:teuthology.orchestra.run.vm08.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 57/140 2026-03-20T11:21:07.876 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libarrow-doc-9.0.0-15.el9.noarch 58/140 2026-03-20T11:21:07.909 INFO:teuthology.orchestra.run.vm08.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 59/140 2026-03-20T11:21:07.967 INFO:teuthology.orchestra.run.vm08.stdout: Installing : abseil-cpp-20211102.0-4.el9.x86_64 60/140 2026-03-20T11:21:07.981 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 61/140 2026-03-20T11:21:08.018 INFO:teuthology.orchestra.run.vm08.stdout: Installing : socat-1.7.4.1-8.el9.x86_64 62/140 2026-03-20T11:21:08.066 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 63/140 2026-03-20T11:21:08.078 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 64/140 2026-03-20T11:21:08.083 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-jaraco-text-4.0.0-2.el9.noarch 65/140 2026-03-20T11:21:08.093 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 66/140 2026-03-20T11:21:08.098 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 67/140 2026-03-20T11:21:08.134 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 68/140 2026-03-20T11:21:08.154 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-protobuf-3.14.0-17.el9.noarch 69/140 2026-03-20T11:21:08.163 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 70/140 2026-03-20T11:21:08.172 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 71/140 2026-03-20T11:21:08.216 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-jmespath-1.0.1-1.el9.noarch 72/140 2026-03-20T11:21:08.497 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.no 122/140 2026-03-20T11:21:08.500 INFO:teuthology.orchestra.run.vm07.stdout: Installing : ceph-mgr-diskprediction-local-2:20.2.0-712.g70f8 123/140 2026-03-20T11:21:08.509 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-devel-3.9.25-3.el9.x86_64 73/140 2026-03-20T11:21:08.541 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 74/140 2026-03-20T11:21:08.545 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-jinja2-2.11.3-8.el9.noarch 75/140 2026-03-20T11:21:08.549 INFO:teuthology.orchestra.run.vm08.stdout: Installing : perl-Benchmark-1.23-483.el9.noarch 76/140 2026-03-20T11:21:08.580 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:20.2.0-712.g70f8 123/140 2026-03-20T11:21:08.619 INFO:teuthology.orchestra.run.vm08.stdout: Installing : openblas-0.3.29-1.el9.x86_64 77/140 2026-03-20T11:21:08.623 INFO:teuthology.orchestra.run.vm08.stdout: Installing : openblas-openmp-0.3.29-1.el9.x86_64 78/140 2026-03-20T11:21:08.640 INFO:teuthology.orchestra.run.vm07.stdout: Installing : ceph-mgr-modules-core-2:20.2.0-712.g70f8415b.el9 124/140 2026-03-20T11:21:08.643 INFO:teuthology.orchestra.run.vm07.stdout: Installing : ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 125/140 2026-03-20T11:21:08.653 INFO:teuthology.orchestra.run.vm08.stdout: Installing : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 79/140 2026-03-20T11:21:08.671 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 125/140 2026-03-20T11:21:08.671 INFO:teuthology.orchestra.run.vm07.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:08.671 INFO:teuthology.orchestra.run.vm07.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2026-03-20T11:21:08.672 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-03-20T11:21:08.672 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-03-20T11:21:08.672 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:08.690 INFO:teuthology.orchestra.run.vm07.stdout: Installing : ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 126/140 2026-03-20T11:21:08.707 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 126/140 2026-03-20T11:21:08.758 INFO:teuthology.orchestra.run.vm07.stdout: Installing : ceph-2:20.2.0-712.g70f8415b.el9.x86_64 127/140 2026-03-20T11:21:08.881 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noar 121/140 2026-03-20T11:21:08.892 INFO:teuthology.orchestra.run.vm05.stdout: Installing : ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.no 122/140 2026-03-20T11:21:09.078 INFO:teuthology.orchestra.run.vm08.stdout: Installing : flexiblas-netlib-3.0.4-9.el9.x86_64 80/140 2026-03-20T11:21:09.175 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-numpy-1:1.23.5-2.el9.x86_64 81/140 2026-03-20T11:21:09.575 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.no 122/140 2026-03-20T11:21:09.577 INFO:teuthology.orchestra.run.vm05.stdout: Installing : ceph-mgr-diskprediction-local-2:20.2.0-712.g70f8 123/140 2026-03-20T11:21:09.657 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:20.2.0-712.g70f8 123/140 2026-03-20T11:21:09.713 INFO:teuthology.orchestra.run.vm05.stdout: Installing : ceph-mgr-modules-core-2:20.2.0-712.g70f8415b.el9 124/140 2026-03-20T11:21:09.717 INFO:teuthology.orchestra.run.vm05.stdout: Installing : ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 125/140 2026-03-20T11:21:09.750 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 125/140 2026-03-20T11:21:09.750 INFO:teuthology.orchestra.run.vm05.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:09.750 INFO:teuthology.orchestra.run.vm05.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2026-03-20T11:21:09.750 INFO:teuthology.orchestra.run.vm05.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-03-20T11:21:09.750 INFO:teuthology.orchestra.run.vm05.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-03-20T11:21:09.750 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:09.771 INFO:teuthology.orchestra.run.vm05.stdout: Installing : ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 126/140 2026-03-20T11:21:09.786 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 126/140 2026-03-20T11:21:09.838 INFO:teuthology.orchestra.run.vm05.stdout: Installing : ceph-2:20.2.0-712.g70f8415b.el9.x86_64 127/140 2026-03-20T11:21:10.104 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 82/140 2026-03-20T11:21:10.192 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-scipy-1.9.3-2.el9.x86_64 83/140 2026-03-20T11:21:10.199 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libxslt-1.1.34-12.el9.x86_64 84/140 2026-03-20T11:21:10.204 INFO:teuthology.orchestra.run.vm08.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 85/140 2026-03-20T11:21:10.212 INFO:teuthology.orchestra.run.vm08.stdout: Installing : boost-program-options-1.75.0-13.el9.x86_64 86/140 2026-03-20T11:21:10.220 INFO:teuthology.orchestra.run.vm07.stdout: Installing : ceph-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 128/140 2026-03-20T11:21:10.224 INFO:teuthology.orchestra.run.vm07.stdout: Installing : ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 129/140 2026-03-20T11:21:10.249 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 129/140 2026-03-20T11:21:10.249 INFO:teuthology.orchestra.run.vm07.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:10.249 INFO:teuthology.orchestra.run.vm07.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2026-03-20T11:21:10.249 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-03-20T11:21:10.249 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-03-20T11:21:10.249 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:10.261 INFO:teuthology.orchestra.run.vm07.stdout: Installing : ceph-immutable-object-cache-2:20.2.0-712.g70f841 130/140 2026-03-20T11:21:10.284 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: ceph-immutable-object-cache-2:20.2.0-712.g70f841 130/140 2026-03-20T11:21:10.284 INFO:teuthology.orchestra.run.vm07.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:10.284 INFO:teuthology.orchestra.run.vm07.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2026-03-20T11:21:10.284 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:10.448 INFO:teuthology.orchestra.run.vm07.stdout: Installing : rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 131/140 2026-03-20T11:21:10.478 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 131/140 2026-03-20T11:21:10.479 INFO:teuthology.orchestra.run.vm07.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:10.479 INFO:teuthology.orchestra.run.vm07.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2026-03-20T11:21:10.479 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-03-20T11:21:10.479 INFO:teuthology.orchestra.run.vm07.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-03-20T11:21:10.479 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:10.542 INFO:teuthology.orchestra.run.vm08.stdout: Installing : parquet-libs-9.0.0-15.el9.x86_64 87/140 2026-03-20T11:21:10.545 INFO:teuthology.orchestra.run.vm08.stdout: Installing : librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 88/140 2026-03-20T11:21:10.570 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 88/140 2026-03-20T11:21:10.571 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-rgw-2:20.2.0-712.g70f8415b.el9.x86_64 89/140 2026-03-20T11:21:11.172 INFO:teuthology.orchestra.run.vm05.stdout: Installing : ceph-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 128/140 2026-03-20T11:21:11.177 INFO:teuthology.orchestra.run.vm05.stdout: Installing : ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 129/140 2026-03-20T11:21:11.205 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 129/140 2026-03-20T11:21:11.205 INFO:teuthology.orchestra.run.vm05.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:11.205 INFO:teuthology.orchestra.run.vm05.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2026-03-20T11:21:11.205 INFO:teuthology.orchestra.run.vm05.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-03-20T11:21:11.205 INFO:teuthology.orchestra.run.vm05.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-03-20T11:21:11.205 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:11.218 INFO:teuthology.orchestra.run.vm05.stdout: Installing : ceph-immutable-object-cache-2:20.2.0-712.g70f841 130/140 2026-03-20T11:21:11.244 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: ceph-immutable-object-cache-2:20.2.0-712.g70f841 130/140 2026-03-20T11:21:11.244 INFO:teuthology.orchestra.run.vm05.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:11.244 INFO:teuthology.orchestra.run.vm05.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2026-03-20T11:21:11.244 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:11.394 INFO:teuthology.orchestra.run.vm05.stdout: Installing : rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 131/140 2026-03-20T11:21:11.421 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 131/140 2026-03-20T11:21:11.421 INFO:teuthology.orchestra.run.vm05.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:11.421 INFO:teuthology.orchestra.run.vm05.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2026-03-20T11:21:11.421 INFO:teuthology.orchestra.run.vm05.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-03-20T11:21:11.421 INFO:teuthology.orchestra.run.vm05.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-03-20T11:21:11.422 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:11.837 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 90/140 2026-03-20T11:21:11.842 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 90/140 2026-03-20T11:21:11.863 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 90/140 2026-03-20T11:21:11.876 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-pyparsing-2.4.7-9.el9.noarch 91/140 2026-03-20T11:21:11.886 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-packaging-20.9-5.el9.noarch 92/140 2026-03-20T11:21:11.903 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-ply-3.11-14.el9.noarch 93/140 2026-03-20T11:21:11.927 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 94/140 2026-03-20T11:21:12.016 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 95/140 2026-03-20T11:21:12.030 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-cryptography-36.0.1-5.el9.x86_64 96/140 2026-03-20T11:21:12.059 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 97/140 2026-03-20T11:21:12.096 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-cheroot-10.0.1-4.el9.noarch 98/140 2026-03-20T11:21:12.158 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 99/140 2026-03-20T11:21:12.167 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 100/140 2026-03-20T11:21:12.173 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 101/140 2026-03-20T11:21:12.179 INFO:teuthology.orchestra.run.vm08.stdout: Installing : pciutils-3.7.0-7.el9.x86_64 102/140 2026-03-20T11:21:12.183 INFO:teuthology.orchestra.run.vm08.stdout: Installing : qatlib-25.08.0-2.el9.x86_64 103/140 2026-03-20T11:21:12.185 INFO:teuthology.orchestra.run.vm08.stdout: Installing : qatlib-service-25.08.0-2.el9.x86_64 104/140 2026-03-20T11:21:12.205 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: qatlib-service-25.08.0-2.el9.x86_64 104/140 2026-03-20T11:21:12.540 INFO:teuthology.orchestra.run.vm08.stdout: Installing : qatzip-libs-1.3.1-1.el9.x86_64 105/140 2026-03-20T11:21:12.546 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 106/140 2026-03-20T11:21:12.597 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 106/140 2026-03-20T11:21:12.597 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2026-03-20T11:21:12.597 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2026-03-20T11:21:12.597 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:12.699 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 107/140 2026-03-20T11:21:13.463 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 107/140 2026-03-20T11:21:13.463 INFO:teuthology.orchestra.run.vm02.stdout:skipping the directory /sys 2026-03-20T11:21:13.463 INFO:teuthology.orchestra.run.vm02.stdout:skipping the directory /proc 2026-03-20T11:21:13.463 INFO:teuthology.orchestra.run.vm02.stdout:skipping the directory /mnt 2026-03-20T11:21:13.463 INFO:teuthology.orchestra.run.vm02.stdout:skipping the directory /var/tmp 2026-03-20T11:21:13.463 INFO:teuthology.orchestra.run.vm02.stdout:skipping the directory /home 2026-03-20T11:21:13.463 INFO:teuthology.orchestra.run.vm02.stdout:skipping the directory /root 2026-03-20T11:21:13.463 INFO:teuthology.orchestra.run.vm02.stdout:skipping the directory /tmp 2026-03-20T11:21:13.463 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:13.585 INFO:teuthology.orchestra.run.vm02.stdout: Installing : ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 108/140 2026-03-20T11:21:13.610 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 108/140 2026-03-20T11:21:13.611 INFO:teuthology.orchestra.run.vm02.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:13.611 INFO:teuthology.orchestra.run.vm02.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2026-03-20T11:21:13.611 INFO:teuthology.orchestra.run.vm02.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-03-20T11:21:13.611 INFO:teuthology.orchestra.run.vm02.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-03-20T11:21:13.611 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:13.857 INFO:teuthology.orchestra.run.vm02.stdout: Installing : ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 109/140 2026-03-20T11:21:13.881 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 109/140 2026-03-20T11:21:13.881 INFO:teuthology.orchestra.run.vm02.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:13.881 INFO:teuthology.orchestra.run.vm02.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2026-03-20T11:21:13.881 INFO:teuthology.orchestra.run.vm02.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-03-20T11:21:13.881 INFO:teuthology.orchestra.run.vm02.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-03-20T11:21:13.881 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:13.890 INFO:teuthology.orchestra.run.vm02.stdout: Installing : mailcap-2.1.49-5.el9.noarch 110/140 2026-03-20T11:21:13.892 INFO:teuthology.orchestra.run.vm02.stdout: Installing : libconfig-1.7.2-9.el9.x86_64 111/140 2026-03-20T11:21:13.912 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 112/140 2026-03-20T11:21:13.912 INFO:teuthology.orchestra.run.vm02.stdout:Creating group 'qat' with GID 994. 2026-03-20T11:21:13.912 INFO:teuthology.orchestra.run.vm02.stdout:Creating group 'libstoragemgmt' with GID 993. 2026-03-20T11:21:13.912 INFO:teuthology.orchestra.run.vm02.stdout:Creating user 'libstoragemgmt' (daemon account for libstoragemgmt) with UID 993 and GID 993. 2026-03-20T11:21:13.912 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:13.922 INFO:teuthology.orchestra.run.vm02.stdout: Installing : libstoragemgmt-1.10.1-1.el9.x86_64 112/140 2026-03-20T11:21:13.951 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 112/140 2026-03-20T11:21:13.951 INFO:teuthology.orchestra.run.vm02.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/libstoragemgmt.service → /usr/lib/systemd/system/libstoragemgmt.service. 2026-03-20T11:21:13.951 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:13.970 INFO:teuthology.orchestra.run.vm02.stdout: Installing : python3-libstoragemgmt-1.10.1-1.el9.x86_64 113/140 2026-03-20T11:21:13.997 INFO:teuthology.orchestra.run.vm02.stdout: Installing : fuse-2.9.9-17.el9.x86_64 114/140 2026-03-20T11:21:14.067 INFO:teuthology.orchestra.run.vm02.stdout: Installing : cryptsetup-2.8.1-3.el9.x86_64 115/140 2026-03-20T11:21:14.072 INFO:teuthology.orchestra.run.vm02.stdout: Installing : ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 116/140 2026-03-20T11:21:14.088 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 116/140 2026-03-20T11:21:14.088 INFO:teuthology.orchestra.run.vm02.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:14.088 INFO:teuthology.orchestra.run.vm02.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2026-03-20T11:21:14.088 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:14.860 INFO:teuthology.orchestra.run.vm02.stdout: Installing : ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 117/140 2026-03-20T11:21:14.884 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 117/140 2026-03-20T11:21:14.885 INFO:teuthology.orchestra.run.vm02.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:14.885 INFO:teuthology.orchestra.run.vm02.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2026-03-20T11:21:14.885 INFO:teuthology.orchestra.run.vm02.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-03-20T11:21:14.885 INFO:teuthology.orchestra.run.vm02.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-03-20T11:21:14.885 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:14.951 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: cephadm-2:20.2.0-712.g70f8415b.el9.noarch 118/140 2026-03-20T11:21:14.954 INFO:teuthology.orchestra.run.vm02.stdout: Installing : cephadm-2:20.2.0-712.g70f8415b.el9.noarch 118/140 2026-03-20T11:21:14.962 INFO:teuthology.orchestra.run.vm02.stdout: Installing : ceph-prometheus-alerts-2:20.2.0-712.g70f8415b.el 119/140 2026-03-20T11:21:14.989 INFO:teuthology.orchestra.run.vm02.stdout: Installing : ceph-grafana-dashboards-2:20.2.0-712.g70f8415b.e 120/140 2026-03-20T11:21:14.992 INFO:teuthology.orchestra.run.vm02.stdout: Installing : ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noar 121/140 2026-03-20T11:21:15.241 INFO:teuthology.orchestra.run.vm07.stdout: Installing : ceph-test-2:20.2.0-712.g70f8415b.el9.x86_64 132/140 2026-03-20T11:21:15.264 INFO:teuthology.orchestra.run.vm07.stdout: Installing : perl-Test-Harness-1:3.42-461.el9.noarch 133/140 2026-03-20T11:21:15.270 INFO:teuthology.orchestra.run.vm07.stdout: Installing : libcephfs-devel-2:20.2.0-712.g70f8415b.el9.x86_6 134/140 2026-03-20T11:21:15.283 INFO:teuthology.orchestra.run.vm07.stdout: Installing : rbd-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 135/140 2026-03-20T11:21:15.317 INFO:teuthology.orchestra.run.vm07.stdout: Installing : rbd-nbd-2:20.2.0-712.g70f8415b.el9.x86_64 136/140 2026-03-20T11:21:15.325 INFO:teuthology.orchestra.run.vm07.stdout: Installing : s3cmd-2.4.0-1.el9.noarch 137/140 2026-03-20T11:21:15.329 INFO:teuthology.orchestra.run.vm07.stdout: Installing : bzip2-1.0.8-11.el9.x86_64 138/140 2026-03-20T11:21:15.329 INFO:teuthology.orchestra.run.vm07.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 139/140 2026-03-20T11:21:15.347 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 139/140 2026-03-20T11:21:15.347 INFO:teuthology.orchestra.run.vm07.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 140/140 2026-03-20T11:21:16.146 INFO:teuthology.orchestra.run.vm05.stdout: Installing : ceph-test-2:20.2.0-712.g70f8415b.el9.x86_64 132/140 2026-03-20T11:21:16.155 INFO:teuthology.orchestra.run.vm05.stdout: Installing : perl-Test-Harness-1:3.42-461.el9.noarch 133/140 2026-03-20T11:21:16.162 INFO:teuthology.orchestra.run.vm05.stdout: Installing : libcephfs-devel-2:20.2.0-712.g70f8415b.el9.x86_6 134/140 2026-03-20T11:21:16.184 INFO:teuthology.orchestra.run.vm05.stdout: Installing : rbd-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 135/140 2026-03-20T11:21:16.210 INFO:teuthology.orchestra.run.vm05.stdout: Installing : rbd-nbd-2:20.2.0-712.g70f8415b.el9.x86_64 136/140 2026-03-20T11:21:16.287 INFO:teuthology.orchestra.run.vm05.stdout: Installing : s3cmd-2.4.0-1.el9.noarch 137/140 2026-03-20T11:21:16.320 INFO:teuthology.orchestra.run.vm05.stdout: Installing : bzip2-1.0.8-11.el9.x86_64 138/140 2026-03-20T11:21:16.320 INFO:teuthology.orchestra.run.vm05.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 139/140 2026-03-20T11:21:16.405 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noar 121/140 2026-03-20T11:21:16.427 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 139/140 2026-03-20T11:21:16.438 INFO:teuthology.orchestra.run.vm05.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 140/140 2026-03-20T11:21:16.496 INFO:teuthology.orchestra.run.vm02.stdout: Installing : ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.no 122/140 2026-03-20T11:21:16.847 INFO:teuthology.orchestra.run.vm07.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 140/140 2026-03-20T11:21:16.847 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : ceph-2:20.2.0-712.g70f8415b.el9.x86_64 1/140 2026-03-20T11:21:16.847 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 2/140 2026-03-20T11:21:16.847 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 3/140 2026-03-20T11:21:16.847 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : ceph-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 4/140 2026-03-20T11:21:16.847 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : ceph-immutable-object-cache-2:20.2.0-712.g70f841 5/140 2026-03-20T11:21:16.847 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 6/140 2026-03-20T11:21:16.847 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 7/140 2026-03-20T11:21:16.847 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 8/140 2026-03-20T11:21:16.847 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 9/140 2026-03-20T11:21:16.847 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 10/140 2026-03-20T11:21:16.847 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 11/140 2026-03-20T11:21:16.847 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : ceph-test-2:20.2.0-712.g70f8415b.el9.x86_64 12/140 2026-03-20T11:21:16.847 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : libcephfs-devel-2:20.2.0-712.g70f8415b.el9.x86_6 13/140 2026-03-20T11:21:16.847 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_ 14/140 2026-03-20T11:21:16.847 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 15/140 2026-03-20T11:21:16.847 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 16/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : librados-devel-2:20.2.0-712.g70f8415b.el9.x86_64 17/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_ 18/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 19/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-ceph-argparse-2:20.2.0-712.g70f8415b.el9 20/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-ceph-common-2:20.2.0-712.g70f8415b.el9.x 21/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-cephfs-2:20.2.0-712.g70f8415b.el9.x86_64 22/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-rados-2:20.2.0-712.g70f8415b.el9.x86_64 23/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-rbd-2:20.2.0-712.g70f8415b.el9.x86_64 24/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-rgw-2:20.2.0-712.g70f8415b.el9.x86_64 25/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : rbd-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 26/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 27/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : rbd-nbd-2:20.2.0-712.g70f8415b.el9.x86_64 28/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : ceph-grafana-dashboards-2:20.2.0-712.g70f8415b.e 29/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noar 30/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.no 31/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : ceph-mgr-diskprediction-local-2:20.2.0-712.g70f8 32/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : ceph-mgr-modules-core-2:20.2.0-712.g70f8415b.el9 33/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 34/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : ceph-prometheus-alerts-2:20.2.0-712.g70f8415b.el 35/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 36/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : cephadm-2:20.2.0-712.g70f8415b.el9.noarch 37/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : bzip2-1.0.8-11.el9.x86_64 38/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : cryptsetup-2.8.1-3.el9.x86_64 39/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : fuse-2.9.9-17.el9.x86_64 40/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : ledmon-libs-1.1.0-3.el9.x86_64 41/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : libconfig-1.7.2-9.el9.x86_64 42/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : libgfortran-11.5.0-14.el9.x86_64 43/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : libquadmath-11.5.0-14.el9.x86_64 44/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : mailcap-2.1.49-5.el9.noarch 45/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : pciutils-3.7.0-7.el9.x86_64 46/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 47/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-cryptography-36.0.1-5.el9.x86_64 48/140 2026-03-20T11:21:16.848 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-ply-3.11-14.el9.noarch 49/140 2026-03-20T11:21:16.850 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 50/140 2026-03-20T11:21:16.850 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-pyparsing-2.4.7-9.el9.noarch 51/140 2026-03-20T11:21:16.850 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-requests-2.25.1-10.el9.noarch 52/140 2026-03-20T11:21:16.850 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-urllib3-1.26.5-7.el9.noarch 53/140 2026-03-20T11:21:16.850 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : unzip-6.0-59.el9.x86_64 54/140 2026-03-20T11:21:16.850 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : zip-3.0-35.el9.x86_64 55/140 2026-03-20T11:21:16.850 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : boost-program-options-1.75.0-13.el9.x86_64 56/140 2026-03-20T11:21:16.850 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : flexiblas-3.0.4-9.el9.x86_64 57/140 2026-03-20T11:21:16.850 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : flexiblas-netlib-3.0.4-9.el9.x86_64 58/140 2026-03-20T11:21:16.850 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 59/140 2026-03-20T11:21:16.850 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : libnbd-1.20.3-4.el9.x86_64 60/140 2026-03-20T11:21:16.850 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 61/140 2026-03-20T11:21:16.850 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 62/140 2026-03-20T11:21:16.850 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 63/140 2026-03-20T11:21:16.850 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : libstoragemgmt-1.10.1-1.el9.x86_64 64/140 2026-03-20T11:21:16.850 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : libxslt-1.1.34-12.el9.x86_64 65/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 66/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : lua-5.4.4-4.el9.x86_64 67/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : openblas-0.3.29-1.el9.x86_64 68/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : openblas-openmp-0.3.29-1.el9.x86_64 69/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : perl-Benchmark-1.23-483.el9.noarch 70/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : perl-Test-Harness-1:3.42-461.el9.noarch 71/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : protobuf-3.14.0-17.el9.x86_64 72/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 73/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-devel-3.9.25-3.el9.x86_64 74/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-jinja2-2.11.3-8.el9.noarch 75/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-jmespath-1.0.1-1.el9.noarch 76/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-libstoragemgmt-1.10.1-1.el9.x86_64 77/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 78/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-numpy-1:1.23.5-2.el9.x86_64 79/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 80/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 81/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-protobuf-3.14.0-17.el9.noarch 82/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-pyasn1-0.4.8-7.el9.noarch 83/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-pyasn1-modules-0.4.8-7.el9.noarch 84/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 85/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-scipy-1.9.3-2.el9.x86_64 86/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 87/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : qatlib-25.08.0-2.el9.x86_64 88/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : qatlib-service-25.08.0-2.el9.x86_64 89/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : qatzip-libs-1.3.1-1.el9.x86_64 90/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : socat-1.7.4.1-8.el9.x86_64 91/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 92/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 93/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : protobuf-compiler-3.14.0-17.el9.x86_64 94/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : abseil-cpp-20211102.0-4.el9.x86_64 95/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 96/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 97/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : libarrow-9.0.0-15.el9.x86_64 98/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : libarrow-doc-9.0.0-15.el9.noarch 99/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : liboath-2.6.12-1.el9.x86_64 100/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 101/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 102/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : parquet-libs-9.0.0-15.el9.x86_64 103/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 104/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-autocommand-2.2.2-8.el9.noarch 105/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-backports-tarfile-1.2.0-1.el9.noarch 106/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 107/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 108/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 109/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-cheroot-10.0.1-4.el9.noarch 110/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 111/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-google-auth-1:2.45.0-1.el9.noarch 112/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 113/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 114/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 115/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 116/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 117/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-jaraco-context-6.0.1-3.el9.noarch 118/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 119/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-jaraco-text-4.0.0-2.el9.noarch 120/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-kubernetes-1:26.1.0-3.el9.noarch 121/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 122/140 2026-03-20T11:21:16.851 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 123/140 2026-03-20T11:21:16.852 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 124/140 2026-03-20T11:21:16.852 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 125/140 2026-03-20T11:21:16.852 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 126/140 2026-03-20T11:21:16.852 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 127/140 2026-03-20T11:21:16.852 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 128/140 2026-03-20T11:21:16.852 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 129/140 2026-03-20T11:21:16.852 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-typing-extensions-4.15.0-1.el9.noarch 130/140 2026-03-20T11:21:16.852 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 131/140 2026-03-20T11:21:16.852 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-xmltodict-0.12.0-15.el9.noarch 132/140 2026-03-20T11:21:16.852 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 133/140 2026-03-20T11:21:16.852 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : re2-1:20211101-20.el9.x86_64 134/140 2026-03-20T11:21:16.852 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : s3cmd-2.4.0-1.el9.noarch 135/140 2026-03-20T11:21:16.852 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : thrift-0.15.0-4.el9.x86_64 136/140 2026-03-20T11:21:16.852 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : librados2-2:20.2.0-712.g70f8415b.el9.x86_64 137/140 2026-03-20T11:21:16.852 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 138/140 2026-03-20T11:21:16.852 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 139/140 2026-03-20T11:21:16.958 INFO:teuthology.orchestra.run.vm07.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 140/140 2026-03-20T11:21:16.958 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:16.958 INFO:teuthology.orchestra.run.vm07.stdout:Upgraded: 2026-03-20T11:21:16.958 INFO:teuthology.orchestra.run.vm07.stdout: librados2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.958 INFO:teuthology.orchestra.run.vm07.stdout: librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.958 INFO:teuthology.orchestra.run.vm07.stdout:Installed: 2026-03-20T11:21:16.958 INFO:teuthology.orchestra.run.vm07.stdout: abseil-cpp-20211102.0-4.el9.x86_64 2026-03-20T11:21:16.958 INFO:teuthology.orchestra.run.vm07.stdout: boost-program-options-1.75.0-13.el9.x86_64 2026-03-20T11:21:16.958 INFO:teuthology.orchestra.run.vm07.stdout: bzip2-1.0.8-11.el9.x86_64 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: ceph-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: ceph-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: ceph-grafana-dashboards-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: ceph-immutable-object-cache-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-diskprediction-local-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-modules-core-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: ceph-prometheus-alerts-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: ceph-test-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: cephadm-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: cryptsetup-2.8.1-3.el9.x86_64 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: flexiblas-3.0.4-9.el9.x86_64 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: flexiblas-netlib-3.0.4-9.el9.x86_64 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: fuse-2.9.9-17.el9.x86_64 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: grpc-data-1.46.7-10.el9.noarch 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: ledmon-libs-1.1.0-3.el9.x86_64 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: libarrow-9.0.0-15.el9.x86_64 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: libarrow-doc-9.0.0-15.el9.noarch 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: libcephfs-devel-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.959 INFO:teuthology.orchestra.run.vm07.stdout: libconfig-1.7.2-9.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: libgfortran-11.5.0-14.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: libnbd-1.20.3-4.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: liboath-2.6.12-1.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: libpmemobj-1.12.1-1.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: libquadmath-11.5.0-14.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: librabbitmq-0.11.0-7.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: librados-devel-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: librdkafka-1.6.1-102.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: libstoragemgmt-1.10.1-1.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: libunwind-1.6.2-1.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: libxslt-1.1.34-12.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: lttng-ust-2.12.0-6.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: lua-5.4.4-4.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: lua-devel-5.4.4-4.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: luarocks-3.9.2-5.el9.noarch 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: mailcap-2.1.49-5.el9.noarch 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: openblas-0.3.29-1.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: openblas-openmp-0.3.29-1.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: parquet-libs-9.0.0-15.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: pciutils-3.7.0-7.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: perl-Benchmark-1.23-483.el9.noarch 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: perl-Test-Harness-1:3.42-461.el9.noarch 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: protobuf-3.14.0-17.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: protobuf-compiler-3.14.0-17.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: python3-autocommand-2.2.2-8.el9.noarch 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: python3-babel-2.9.1-2.el9.noarch 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: python3-backports-tarfile-1.2.0-1.el9.noarch 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: python3-cachetools-4.2.4-1.el9.noarch 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: python3-ceph-argparse-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: python3-ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: python3-cephfs-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: python3-certifi-2023.05.07-4.el9.noarch 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: python3-cffi-1.14.5-5.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: python3-cheroot-10.0.1-4.el9.noarch 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: python3-cryptography-36.0.1-5.el9.x86_64 2026-03-20T11:21:16.960 INFO:teuthology.orchestra.run.vm07.stdout: python3-devel-3.9.25-3.el9.x86_64 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-google-auth-1:2.45.0-1.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco-8.2.1-3.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco-context-6.0.1-3.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-jaraco-text-4.0.0-2.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-jinja2-2.11.3-8.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-jmespath-1.0.1-1.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-kubernetes-1:26.1.0-3.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-libstoragemgmt-1.10.1-1.el9.x86_64 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-natsort-7.1.1-5.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-numpy-1:1.23.5-2.el9.x86_64 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-numpy-f2py-1:1.23.5-2.el9.x86_64 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-packaging-20.9-5.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-ply-3.11-14.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-portend-3.1.0-2.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-protobuf-3.14.0-17.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-pyasn1-0.4.8-7.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-pyasn1-modules-0.4.8-7.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-pycparser-2.20-6.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-pyparsing-2.4.7-9.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-rados-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-rbd-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-repoze-lru-0.7-16.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-requests-2.25.1-10.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-rgw-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-routes-2.5.1-5.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-rsa-4.9-2.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-scipy-1.9.3-2.el9.x86_64 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-tempora-5.0.0-2.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-toml-0.10.2-6.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-typing-extensions-4.15.0-1.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-urllib3-1.26.5-7.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-xmltodict-0.12.0-15.el9.noarch 2026-03-20T11:21:16.961 INFO:teuthology.orchestra.run.vm07.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2026-03-20T11:21:16.962 INFO:teuthology.orchestra.run.vm07.stdout: qatlib-25.08.0-2.el9.x86_64 2026-03-20T11:21:16.962 INFO:teuthology.orchestra.run.vm07.stdout: qatlib-service-25.08.0-2.el9.x86_64 2026-03-20T11:21:16.962 INFO:teuthology.orchestra.run.vm07.stdout: qatzip-libs-1.3.1-1.el9.x86_64 2026-03-20T11:21:16.962 INFO:teuthology.orchestra.run.vm07.stdout: rbd-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.962 INFO:teuthology.orchestra.run.vm07.stdout: rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.962 INFO:teuthology.orchestra.run.vm07.stdout: rbd-nbd-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:16.962 INFO:teuthology.orchestra.run.vm07.stdout: re2-1:20211101-20.el9.x86_64 2026-03-20T11:21:16.962 INFO:teuthology.orchestra.run.vm07.stdout: s3cmd-2.4.0-1.el9.noarch 2026-03-20T11:21:16.962 INFO:teuthology.orchestra.run.vm07.stdout: socat-1.7.4.1-8.el9.x86_64 2026-03-20T11:21:16.962 INFO:teuthology.orchestra.run.vm07.stdout: thrift-0.15.0-4.el9.x86_64 2026-03-20T11:21:16.962 INFO:teuthology.orchestra.run.vm07.stdout: unzip-6.0-59.el9.x86_64 2026-03-20T11:21:16.962 INFO:teuthology.orchestra.run.vm07.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2026-03-20T11:21:16.962 INFO:teuthology.orchestra.run.vm07.stdout: zip-3.0-35.el9.x86_64 2026-03-20T11:21:16.962 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:16.962 INFO:teuthology.orchestra.run.vm07.stdout:Complete! 2026-03-20T11:21:17.074 DEBUG:teuthology.parallel:result is None 2026-03-20T11:21:17.168 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.no 122/140 2026-03-20T11:21:17.172 INFO:teuthology.orchestra.run.vm02.stdout: Installing : ceph-mgr-diskprediction-local-2:20.2.0-712.g70f8 123/140 2026-03-20T11:21:17.245 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:20.2.0-712.g70f8 123/140 2026-03-20T11:21:17.307 INFO:teuthology.orchestra.run.vm02.stdout: Installing : ceph-mgr-modules-core-2:20.2.0-712.g70f8415b.el9 124/140 2026-03-20T11:21:17.383 INFO:teuthology.orchestra.run.vm02.stdout: Installing : ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 125/140 2026-03-20T11:21:17.405 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 125/140 2026-03-20T11:21:17.405 INFO:teuthology.orchestra.run.vm02.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:17.405 INFO:teuthology.orchestra.run.vm02.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2026-03-20T11:21:17.405 INFO:teuthology.orchestra.run.vm02.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-03-20T11:21:17.405 INFO:teuthology.orchestra.run.vm02.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-03-20T11:21:17.405 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:17.566 INFO:teuthology.orchestra.run.vm02.stdout: Installing : ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 126/140 2026-03-20T11:21:17.579 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 126/140 2026-03-20T11:21:17.655 INFO:teuthology.orchestra.run.vm02.stdout: Installing : ceph-2:20.2.0-712.g70f8415b.el9.x86_64 127/140 2026-03-20T11:21:18.031 INFO:teuthology.orchestra.run.vm05.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 140/140 2026-03-20T11:21:18.031 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : ceph-2:20.2.0-712.g70f8415b.el9.x86_64 1/140 2026-03-20T11:21:18.031 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 2/140 2026-03-20T11:21:18.031 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 3/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : ceph-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 4/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : ceph-immutable-object-cache-2:20.2.0-712.g70f841 5/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 6/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 7/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 8/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 9/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 10/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 11/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : ceph-test-2:20.2.0-712.g70f8415b.el9.x86_64 12/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : libcephfs-devel-2:20.2.0-712.g70f8415b.el9.x86_6 13/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_ 14/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 15/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 16/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : librados-devel-2:20.2.0-712.g70f8415b.el9.x86_64 17/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_ 18/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 19/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-ceph-argparse-2:20.2.0-712.g70f8415b.el9 20/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-ceph-common-2:20.2.0-712.g70f8415b.el9.x 21/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-cephfs-2:20.2.0-712.g70f8415b.el9.x86_64 22/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-rados-2:20.2.0-712.g70f8415b.el9.x86_64 23/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-rbd-2:20.2.0-712.g70f8415b.el9.x86_64 24/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-rgw-2:20.2.0-712.g70f8415b.el9.x86_64 25/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : rbd-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 26/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 27/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : rbd-nbd-2:20.2.0-712.g70f8415b.el9.x86_64 28/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : ceph-grafana-dashboards-2:20.2.0-712.g70f8415b.e 29/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noar 30/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.no 31/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : ceph-mgr-diskprediction-local-2:20.2.0-712.g70f8 32/140 2026-03-20T11:21:18.032 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : ceph-mgr-modules-core-2:20.2.0-712.g70f8415b.el9 33/140 2026-03-20T11:21:18.034 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 34/140 2026-03-20T11:21:18.034 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : ceph-prometheus-alerts-2:20.2.0-712.g70f8415b.el 35/140 2026-03-20T11:21:18.034 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 36/140 2026-03-20T11:21:18.034 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : cephadm-2:20.2.0-712.g70f8415b.el9.noarch 37/140 2026-03-20T11:21:18.034 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : bzip2-1.0.8-11.el9.x86_64 38/140 2026-03-20T11:21:18.034 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : cryptsetup-2.8.1-3.el9.x86_64 39/140 2026-03-20T11:21:18.034 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : fuse-2.9.9-17.el9.x86_64 40/140 2026-03-20T11:21:18.034 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : ledmon-libs-1.1.0-3.el9.x86_64 41/140 2026-03-20T11:21:18.034 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : libconfig-1.7.2-9.el9.x86_64 42/140 2026-03-20T11:21:18.034 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : libgfortran-11.5.0-14.el9.x86_64 43/140 2026-03-20T11:21:18.034 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : libquadmath-11.5.0-14.el9.x86_64 44/140 2026-03-20T11:21:18.034 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : mailcap-2.1.49-5.el9.noarch 45/140 2026-03-20T11:21:18.034 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : pciutils-3.7.0-7.el9.x86_64 46/140 2026-03-20T11:21:18.034 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 47/140 2026-03-20T11:21:18.034 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-cryptography-36.0.1-5.el9.x86_64 48/140 2026-03-20T11:21:18.034 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-ply-3.11-14.el9.noarch 49/140 2026-03-20T11:21:18.034 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 50/140 2026-03-20T11:21:18.034 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-pyparsing-2.4.7-9.el9.noarch 51/140 2026-03-20T11:21:18.034 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-requests-2.25.1-10.el9.noarch 52/140 2026-03-20T11:21:18.034 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-urllib3-1.26.5-7.el9.noarch 53/140 2026-03-20T11:21:18.034 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : unzip-6.0-59.el9.x86_64 54/140 2026-03-20T11:21:18.034 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : zip-3.0-35.el9.x86_64 55/140 2026-03-20T11:21:18.034 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : boost-program-options-1.75.0-13.el9.x86_64 56/140 2026-03-20T11:21:18.034 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : flexiblas-3.0.4-9.el9.x86_64 57/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : flexiblas-netlib-3.0.4-9.el9.x86_64 58/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 59/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : libnbd-1.20.3-4.el9.x86_64 60/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 61/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 62/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 63/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : libstoragemgmt-1.10.1-1.el9.x86_64 64/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : libxslt-1.1.34-12.el9.x86_64 65/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 66/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : lua-5.4.4-4.el9.x86_64 67/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : openblas-0.3.29-1.el9.x86_64 68/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : openblas-openmp-0.3.29-1.el9.x86_64 69/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : perl-Benchmark-1.23-483.el9.noarch 70/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : perl-Test-Harness-1:3.42-461.el9.noarch 71/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : protobuf-3.14.0-17.el9.x86_64 72/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 73/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-devel-3.9.25-3.el9.x86_64 74/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-jinja2-2.11.3-8.el9.noarch 75/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-jmespath-1.0.1-1.el9.noarch 76/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-libstoragemgmt-1.10.1-1.el9.x86_64 77/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 78/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-numpy-1:1.23.5-2.el9.x86_64 79/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 80/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 81/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-protobuf-3.14.0-17.el9.noarch 82/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-pyasn1-0.4.8-7.el9.noarch 83/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-pyasn1-modules-0.4.8-7.el9.noarch 84/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 85/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-scipy-1.9.3-2.el9.x86_64 86/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 87/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : qatlib-25.08.0-2.el9.x86_64 88/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : qatlib-service-25.08.0-2.el9.x86_64 89/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : qatzip-libs-1.3.1-1.el9.x86_64 90/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : socat-1.7.4.1-8.el9.x86_64 91/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 92/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 93/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : protobuf-compiler-3.14.0-17.el9.x86_64 94/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : abseil-cpp-20211102.0-4.el9.x86_64 95/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 96/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 97/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : libarrow-9.0.0-15.el9.x86_64 98/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : libarrow-doc-9.0.0-15.el9.noarch 99/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : liboath-2.6.12-1.el9.x86_64 100/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 101/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 102/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : parquet-libs-9.0.0-15.el9.x86_64 103/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 104/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-autocommand-2.2.2-8.el9.noarch 105/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-backports-tarfile-1.2.0-1.el9.noarch 106/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 107/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 108/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 109/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-cheroot-10.0.1-4.el9.noarch 110/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 111/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-google-auth-1:2.45.0-1.el9.noarch 112/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 113/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 114/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 115/140 2026-03-20T11:21:18.035 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 116/140 2026-03-20T11:21:18.036 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 117/140 2026-03-20T11:21:18.036 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-jaraco-context-6.0.1-3.el9.noarch 118/140 2026-03-20T11:21:18.036 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 119/140 2026-03-20T11:21:18.036 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-jaraco-text-4.0.0-2.el9.noarch 120/140 2026-03-20T11:21:18.036 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-kubernetes-1:26.1.0-3.el9.noarch 121/140 2026-03-20T11:21:18.036 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 122/140 2026-03-20T11:21:18.036 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 123/140 2026-03-20T11:21:18.036 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 124/140 2026-03-20T11:21:18.036 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 125/140 2026-03-20T11:21:18.036 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 126/140 2026-03-20T11:21:18.036 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 127/140 2026-03-20T11:21:18.036 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 128/140 2026-03-20T11:21:18.036 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 129/140 2026-03-20T11:21:18.036 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-typing-extensions-4.15.0-1.el9.noarch 130/140 2026-03-20T11:21:18.036 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 131/140 2026-03-20T11:21:18.036 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-xmltodict-0.12.0-15.el9.noarch 132/140 2026-03-20T11:21:18.036 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 133/140 2026-03-20T11:21:18.036 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : re2-1:20211101-20.el9.x86_64 134/140 2026-03-20T11:21:18.036 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : s3cmd-2.4.0-1.el9.noarch 135/140 2026-03-20T11:21:18.036 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : thrift-0.15.0-4.el9.x86_64 136/140 2026-03-20T11:21:18.036 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : librados2-2:20.2.0-712.g70f8415b.el9.x86_64 137/140 2026-03-20T11:21:18.036 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 138/140 2026-03-20T11:21:18.036 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 139/140 2026-03-20T11:21:18.143 INFO:teuthology.orchestra.run.vm05.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 140/140 2026-03-20T11:21:18.143 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:18.143 INFO:teuthology.orchestra.run.vm05.stdout:Upgraded: 2026-03-20T11:21:18.143 INFO:teuthology.orchestra.run.vm05.stdout: librados2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.143 INFO:teuthology.orchestra.run.vm05.stdout: librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.143 INFO:teuthology.orchestra.run.vm05.stdout:Installed: 2026-03-20T11:21:18.143 INFO:teuthology.orchestra.run.vm05.stdout: abseil-cpp-20211102.0-4.el9.x86_64 2026-03-20T11:21:18.143 INFO:teuthology.orchestra.run.vm05.stdout: boost-program-options-1.75.0-13.el9.x86_64 2026-03-20T11:21:18.143 INFO:teuthology.orchestra.run.vm05.stdout: bzip2-1.0.8-11.el9.x86_64 2026-03-20T11:21:18.143 INFO:teuthology.orchestra.run.vm05.stdout: ceph-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.143 INFO:teuthology.orchestra.run.vm05.stdout: ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.143 INFO:teuthology.orchestra.run.vm05.stdout: ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: ceph-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: ceph-grafana-dashboards-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: ceph-immutable-object-cache-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: ceph-mgr-diskprediction-local-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: ceph-mgr-modules-core-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: ceph-prometheus-alerts-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: ceph-test-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: cephadm-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: cryptsetup-2.8.1-3.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: flexiblas-3.0.4-9.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: flexiblas-netlib-3.0.4-9.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: fuse-2.9.9-17.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: grpc-data-1.46.7-10.el9.noarch 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: ledmon-libs-1.1.0-3.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: libarrow-9.0.0-15.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: libarrow-doc-9.0.0-15.el9.noarch 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: libcephfs-devel-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: libconfig-1.7.2-9.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: libgfortran-11.5.0-14.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: libnbd-1.20.3-4.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: liboath-2.6.12-1.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: libpmemobj-1.12.1-1.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: libquadmath-11.5.0-14.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: librabbitmq-0.11.0-7.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: librados-devel-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: librdkafka-1.6.1-102.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.144 INFO:teuthology.orchestra.run.vm05.stdout: libstoragemgmt-1.10.1-1.el9.x86_64 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: libunwind-1.6.2-1.el9.x86_64 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: libxslt-1.1.34-12.el9.x86_64 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: lttng-ust-2.12.0-6.el9.x86_64 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: lua-5.4.4-4.el9.x86_64 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: lua-devel-5.4.4-4.el9.x86_64 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: luarocks-3.9.2-5.el9.noarch 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: mailcap-2.1.49-5.el9.noarch 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: openblas-0.3.29-1.el9.x86_64 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: openblas-openmp-0.3.29-1.el9.x86_64 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: parquet-libs-9.0.0-15.el9.x86_64 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: pciutils-3.7.0-7.el9.x86_64 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: perl-Benchmark-1.23-483.el9.noarch 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: perl-Test-Harness-1:3.42-461.el9.noarch 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: protobuf-3.14.0-17.el9.x86_64 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: protobuf-compiler-3.14.0-17.el9.x86_64 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-autocommand-2.2.2-8.el9.noarch 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-babel-2.9.1-2.el9.noarch 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-backports-tarfile-1.2.0-1.el9.noarch 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-cachetools-4.2.4-1.el9.noarch 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-ceph-argparse-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-cephfs-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-certifi-2023.05.07-4.el9.noarch 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-cffi-1.14.5-5.el9.x86_64 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-cheroot-10.0.1-4.el9.noarch 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-cryptography-36.0.1-5.el9.x86_64 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-devel-3.9.25-3.el9.x86_64 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-google-auth-1:2.45.0-1.el9.noarch 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-jaraco-8.2.1-3.el9.noarch 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-jaraco-context-6.0.1-3.el9.noarch 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-jaraco-text-4.0.0-2.el9.noarch 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-jinja2-2.11.3-8.el9.noarch 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-jmespath-1.0.1-1.el9.noarch 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-kubernetes-1:26.1.0-3.el9.noarch 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-libstoragemgmt-1.10.1-1.el9.x86_64 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2026-03-20T11:21:18.145 INFO:teuthology.orchestra.run.vm05.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: python3-natsort-7.1.1-5.el9.noarch 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: python3-numpy-1:1.23.5-2.el9.x86_64 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: python3-numpy-f2py-1:1.23.5-2.el9.x86_64 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: python3-packaging-20.9-5.el9.noarch 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: python3-ply-3.11-14.el9.noarch 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: python3-portend-3.1.0-2.el9.noarch 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: python3-protobuf-3.14.0-17.el9.noarch 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: python3-pyasn1-0.4.8-7.el9.noarch 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: python3-pyasn1-modules-0.4.8-7.el9.noarch 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: python3-pycparser-2.20-6.el9.noarch 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: python3-pyparsing-2.4.7-9.el9.noarch 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: python3-rados-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: python3-rbd-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: python3-repoze-lru-0.7-16.el9.noarch 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: python3-requests-2.25.1-10.el9.noarch 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: python3-rgw-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: python3-routes-2.5.1-5.el9.noarch 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: python3-rsa-4.9-2.el9.noarch 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: python3-scipy-1.9.3-2.el9.x86_64 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: python3-tempora-5.0.0-2.el9.noarch 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: python3-toml-0.10.2-6.el9.noarch 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: python3-typing-extensions-4.15.0-1.el9.noarch 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: python3-urllib3-1.26.5-7.el9.noarch 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: python3-xmltodict-0.12.0-15.el9.noarch 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: qatlib-25.08.0-2.el9.x86_64 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: qatlib-service-25.08.0-2.el9.x86_64 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: qatzip-libs-1.3.1-1.el9.x86_64 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: rbd-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: rbd-nbd-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: re2-1:20211101-20.el9.x86_64 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: s3cmd-2.4.0-1.el9.noarch 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: socat-1.7.4.1-8.el9.x86_64 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: thrift-0.15.0-4.el9.x86_64 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: unzip-6.0-59.el9.x86_64 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: zip-3.0-35.el9.x86_64 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:18.146 INFO:teuthology.orchestra.run.vm05.stdout:Complete! 2026-03-20T11:21:18.249 DEBUG:teuthology.parallel:result is None 2026-03-20T11:21:18.987 INFO:teuthology.orchestra.run.vm02.stdout: Installing : ceph-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 128/140 2026-03-20T11:21:18.992 INFO:teuthology.orchestra.run.vm02.stdout: Installing : ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 129/140 2026-03-20T11:21:19.014 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 129/140 2026-03-20T11:21:19.015 INFO:teuthology.orchestra.run.vm02.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:19.015 INFO:teuthology.orchestra.run.vm02.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2026-03-20T11:21:19.015 INFO:teuthology.orchestra.run.vm02.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-03-20T11:21:19.015 INFO:teuthology.orchestra.run.vm02.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-03-20T11:21:19.015 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:19.027 INFO:teuthology.orchestra.run.vm02.stdout: Installing : ceph-immutable-object-cache-2:20.2.0-712.g70f841 130/140 2026-03-20T11:21:19.049 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: ceph-immutable-object-cache-2:20.2.0-712.g70f841 130/140 2026-03-20T11:21:19.049 INFO:teuthology.orchestra.run.vm02.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:19.049 INFO:teuthology.orchestra.run.vm02.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2026-03-20T11:21:19.049 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:19.108 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 107/140 2026-03-20T11:21:19.108 INFO:teuthology.orchestra.run.vm08.stdout:skipping the directory /sys 2026-03-20T11:21:19.108 INFO:teuthology.orchestra.run.vm08.stdout:skipping the directory /proc 2026-03-20T11:21:19.108 INFO:teuthology.orchestra.run.vm08.stdout:skipping the directory /mnt 2026-03-20T11:21:19.108 INFO:teuthology.orchestra.run.vm08.stdout:skipping the directory /var/tmp 2026-03-20T11:21:19.108 INFO:teuthology.orchestra.run.vm08.stdout:skipping the directory /home 2026-03-20T11:21:19.108 INFO:teuthology.orchestra.run.vm08.stdout:skipping the directory /root 2026-03-20T11:21:19.108 INFO:teuthology.orchestra.run.vm08.stdout:skipping the directory /tmp 2026-03-20T11:21:19.109 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:19.207 INFO:teuthology.orchestra.run.vm02.stdout: Installing : rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 131/140 2026-03-20T11:21:19.224 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 108/140 2026-03-20T11:21:19.228 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 131/140 2026-03-20T11:21:19.228 INFO:teuthology.orchestra.run.vm02.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:19.228 INFO:teuthology.orchestra.run.vm02.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2026-03-20T11:21:19.228 INFO:teuthology.orchestra.run.vm02.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-03-20T11:21:19.228 INFO:teuthology.orchestra.run.vm02.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-03-20T11:21:19.228 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:19.246 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 108/140 2026-03-20T11:21:19.246 INFO:teuthology.orchestra.run.vm08.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:19.246 INFO:teuthology.orchestra.run.vm08.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2026-03-20T11:21:19.246 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-03-20T11:21:19.247 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-03-20T11:21:19.247 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:19.483 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 109/140 2026-03-20T11:21:19.503 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 109/140 2026-03-20T11:21:19.503 INFO:teuthology.orchestra.run.vm08.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:19.503 INFO:teuthology.orchestra.run.vm08.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2026-03-20T11:21:19.503 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-03-20T11:21:19.503 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-03-20T11:21:19.503 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:19.511 INFO:teuthology.orchestra.run.vm08.stdout: Installing : mailcap-2.1.49-5.el9.noarch 110/140 2026-03-20T11:21:19.514 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libconfig-1.7.2-9.el9.x86_64 111/140 2026-03-20T11:21:19.532 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 112/140 2026-03-20T11:21:19.532 INFO:teuthology.orchestra.run.vm08.stdout:Creating group 'qat' with GID 994. 2026-03-20T11:21:19.532 INFO:teuthology.orchestra.run.vm08.stdout:Creating group 'libstoragemgmt' with GID 993. 2026-03-20T11:21:19.532 INFO:teuthology.orchestra.run.vm08.stdout:Creating user 'libstoragemgmt' (daemon account for libstoragemgmt) with UID 993 and GID 993. 2026-03-20T11:21:19.532 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:19.542 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libstoragemgmt-1.10.1-1.el9.x86_64 112/140 2026-03-20T11:21:19.567 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 112/140 2026-03-20T11:21:19.567 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/libstoragemgmt.service → /usr/lib/systemd/system/libstoragemgmt.service. 2026-03-20T11:21:19.567 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:19.587 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-libstoragemgmt-1.10.1-1.el9.x86_64 113/140 2026-03-20T11:21:19.616 INFO:teuthology.orchestra.run.vm08.stdout: Installing : fuse-2.9.9-17.el9.x86_64 114/140 2026-03-20T11:21:19.690 INFO:teuthology.orchestra.run.vm08.stdout: Installing : cryptsetup-2.8.1-3.el9.x86_64 115/140 2026-03-20T11:21:19.695 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 116/140 2026-03-20T11:21:19.709 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 116/140 2026-03-20T11:21:19.709 INFO:teuthology.orchestra.run.vm08.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:19.709 INFO:teuthology.orchestra.run.vm08.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2026-03-20T11:21:19.709 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:20.478 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 117/140 2026-03-20T11:21:20.500 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 117/140 2026-03-20T11:21:20.501 INFO:teuthology.orchestra.run.vm08.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:20.501 INFO:teuthology.orchestra.run.vm08.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2026-03-20T11:21:20.501 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-03-20T11:21:20.501 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-03-20T11:21:20.501 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:20.567 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: cephadm-2:20.2.0-712.g70f8415b.el9.noarch 118/140 2026-03-20T11:21:20.571 INFO:teuthology.orchestra.run.vm08.stdout: Installing : cephadm-2:20.2.0-712.g70f8415b.el9.noarch 118/140 2026-03-20T11:21:20.579 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-prometheus-alerts-2:20.2.0-712.g70f8415b.el 119/140 2026-03-20T11:21:20.604 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-grafana-dashboards-2:20.2.0-712.g70f8415b.e 120/140 2026-03-20T11:21:20.607 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noar 121/140 2026-03-20T11:21:21.852 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noar 121/140 2026-03-20T11:21:21.862 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.no 122/140 2026-03-20T11:21:22.385 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.no 122/140 2026-03-20T11:21:22.387 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-mgr-diskprediction-local-2:20.2.0-712.g70f8 123/140 2026-03-20T11:21:22.449 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:20.2.0-712.g70f8 123/140 2026-03-20T11:21:22.498 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-mgr-modules-core-2:20.2.0-712.g70f8415b.el9 124/140 2026-03-20T11:21:22.500 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 125/140 2026-03-20T11:21:22.521 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 125/140 2026-03-20T11:21:22.521 INFO:teuthology.orchestra.run.vm08.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:22.521 INFO:teuthology.orchestra.run.vm08.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2026-03-20T11:21:22.521 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-03-20T11:21:22.521 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-03-20T11:21:22.521 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:22.535 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 126/140 2026-03-20T11:21:22.546 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 126/140 2026-03-20T11:21:22.591 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-2:20.2.0-712.g70f8415b.el9.x86_64 127/140 2026-03-20T11:21:23.846 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 128/140 2026-03-20T11:21:23.851 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 129/140 2026-03-20T11:21:23.872 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 129/140 2026-03-20T11:21:23.872 INFO:teuthology.orchestra.run.vm08.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:23.872 INFO:teuthology.orchestra.run.vm08.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2026-03-20T11:21:23.872 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-03-20T11:21:23.872 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-03-20T11:21:23.872 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:23.884 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-immutable-object-cache-2:20.2.0-712.g70f841 130/140 2026-03-20T11:21:23.904 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-immutable-object-cache-2:20.2.0-712.g70f841 130/140 2026-03-20T11:21:23.904 INFO:teuthology.orchestra.run.vm08.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:23.904 INFO:teuthology.orchestra.run.vm08.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2026-03-20T11:21:23.904 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:23.974 INFO:teuthology.orchestra.run.vm02.stdout: Installing : ceph-test-2:20.2.0-712.g70f8415b.el9.x86_64 132/140 2026-03-20T11:21:23.982 INFO:teuthology.orchestra.run.vm02.stdout: Installing : perl-Test-Harness-1:3.42-461.el9.noarch 133/140 2026-03-20T11:21:23.988 INFO:teuthology.orchestra.run.vm02.stdout: Installing : libcephfs-devel-2:20.2.0-712.g70f8415b.el9.x86_6 134/140 2026-03-20T11:21:24.000 INFO:teuthology.orchestra.run.vm02.stdout: Installing : rbd-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 135/140 2026-03-20T11:21:24.023 INFO:teuthology.orchestra.run.vm02.stdout: Installing : rbd-nbd-2:20.2.0-712.g70f8415b.el9.x86_64 136/140 2026-03-20T11:21:24.031 INFO:teuthology.orchestra.run.vm02.stdout: Installing : s3cmd-2.4.0-1.el9.noarch 137/140 2026-03-20T11:21:24.036 INFO:teuthology.orchestra.run.vm02.stdout: Installing : bzip2-1.0.8-11.el9.x86_64 138/140 2026-03-20T11:21:24.036 INFO:teuthology.orchestra.run.vm02.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 139/140 2026-03-20T11:21:24.051 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 139/140 2026-03-20T11:21:24.052 INFO:teuthology.orchestra.run.vm02.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 140/140 2026-03-20T11:21:24.058 INFO:teuthology.orchestra.run.vm08.stdout: Installing : rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 131/140 2026-03-20T11:21:24.078 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 131/140 2026-03-20T11:21:24.079 INFO:teuthology.orchestra.run.vm08.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-20T11:21:24.079 INFO:teuthology.orchestra.run.vm08.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2026-03-20T11:21:24.079 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-03-20T11:21:24.079 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-03-20T11:21:24.079 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:25.476 INFO:teuthology.orchestra.run.vm02.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 140/140 2026-03-20T11:21:25.476 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : ceph-2:20.2.0-712.g70f8415b.el9.x86_64 1/140 2026-03-20T11:21:25.476 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 2/140 2026-03-20T11:21:25.476 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 3/140 2026-03-20T11:21:25.476 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : ceph-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 4/140 2026-03-20T11:21:25.476 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : ceph-immutable-object-cache-2:20.2.0-712.g70f841 5/140 2026-03-20T11:21:25.476 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 6/140 2026-03-20T11:21:25.476 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 7/140 2026-03-20T11:21:25.476 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 8/140 2026-03-20T11:21:25.476 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 9/140 2026-03-20T11:21:25.476 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 10/140 2026-03-20T11:21:25.476 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 11/140 2026-03-20T11:21:25.476 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : ceph-test-2:20.2.0-712.g70f8415b.el9.x86_64 12/140 2026-03-20T11:21:25.477 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : libcephfs-devel-2:20.2.0-712.g70f8415b.el9.x86_6 13/140 2026-03-20T11:21:25.477 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_ 14/140 2026-03-20T11:21:25.477 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 15/140 2026-03-20T11:21:25.477 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 16/140 2026-03-20T11:21:25.477 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : librados-devel-2:20.2.0-712.g70f8415b.el9.x86_64 17/140 2026-03-20T11:21:25.477 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_ 18/140 2026-03-20T11:21:25.477 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 19/140 2026-03-20T11:21:25.477 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-ceph-argparse-2:20.2.0-712.g70f8415b.el9 20/140 2026-03-20T11:21:25.477 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-ceph-common-2:20.2.0-712.g70f8415b.el9.x 21/140 2026-03-20T11:21:25.477 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-cephfs-2:20.2.0-712.g70f8415b.el9.x86_64 22/140 2026-03-20T11:21:25.477 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-rados-2:20.2.0-712.g70f8415b.el9.x86_64 23/140 2026-03-20T11:21:25.477 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-rbd-2:20.2.0-712.g70f8415b.el9.x86_64 24/140 2026-03-20T11:21:25.477 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-rgw-2:20.2.0-712.g70f8415b.el9.x86_64 25/140 2026-03-20T11:21:25.477 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : rbd-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 26/140 2026-03-20T11:21:25.477 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 27/140 2026-03-20T11:21:25.477 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : rbd-nbd-2:20.2.0-712.g70f8415b.el9.x86_64 28/140 2026-03-20T11:21:25.477 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : ceph-grafana-dashboards-2:20.2.0-712.g70f8415b.e 29/140 2026-03-20T11:21:25.477 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noar 30/140 2026-03-20T11:21:25.477 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.no 31/140 2026-03-20T11:21:25.477 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : ceph-mgr-diskprediction-local-2:20.2.0-712.g70f8 32/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : ceph-mgr-modules-core-2:20.2.0-712.g70f8415b.el9 33/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 34/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : ceph-prometheus-alerts-2:20.2.0-712.g70f8415b.el 35/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 36/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : cephadm-2:20.2.0-712.g70f8415b.el9.noarch 37/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : bzip2-1.0.8-11.el9.x86_64 38/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : cryptsetup-2.8.1-3.el9.x86_64 39/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : fuse-2.9.9-17.el9.x86_64 40/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : ledmon-libs-1.1.0-3.el9.x86_64 41/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : libconfig-1.7.2-9.el9.x86_64 42/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : libgfortran-11.5.0-14.el9.x86_64 43/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : libquadmath-11.5.0-14.el9.x86_64 44/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : mailcap-2.1.49-5.el9.noarch 45/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : pciutils-3.7.0-7.el9.x86_64 46/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 47/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-cryptography-36.0.1-5.el9.x86_64 48/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-ply-3.11-14.el9.noarch 49/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 50/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-pyparsing-2.4.7-9.el9.noarch 51/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-requests-2.25.1-10.el9.noarch 52/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-urllib3-1.26.5-7.el9.noarch 53/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : unzip-6.0-59.el9.x86_64 54/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : zip-3.0-35.el9.x86_64 55/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : boost-program-options-1.75.0-13.el9.x86_64 56/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : flexiblas-3.0.4-9.el9.x86_64 57/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : flexiblas-netlib-3.0.4-9.el9.x86_64 58/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 59/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : libnbd-1.20.3-4.el9.x86_64 60/140 2026-03-20T11:21:25.478 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 61/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 62/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 63/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : libstoragemgmt-1.10.1-1.el9.x86_64 64/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : libxslt-1.1.34-12.el9.x86_64 65/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 66/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : lua-5.4.4-4.el9.x86_64 67/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : openblas-0.3.29-1.el9.x86_64 68/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : openblas-openmp-0.3.29-1.el9.x86_64 69/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : perl-Benchmark-1.23-483.el9.noarch 70/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : perl-Test-Harness-1:3.42-461.el9.noarch 71/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : protobuf-3.14.0-17.el9.x86_64 72/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 73/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-devel-3.9.25-3.el9.x86_64 74/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-jinja2-2.11.3-8.el9.noarch 75/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-jmespath-1.0.1-1.el9.noarch 76/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-libstoragemgmt-1.10.1-1.el9.x86_64 77/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 78/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-numpy-1:1.23.5-2.el9.x86_64 79/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 80/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 81/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-protobuf-3.14.0-17.el9.noarch 82/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-pyasn1-0.4.8-7.el9.noarch 83/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-pyasn1-modules-0.4.8-7.el9.noarch 84/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 85/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-scipy-1.9.3-2.el9.x86_64 86/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 87/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : qatlib-25.08.0-2.el9.x86_64 88/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : qatlib-service-25.08.0-2.el9.x86_64 89/140 2026-03-20T11:21:25.479 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : qatzip-libs-1.3.1-1.el9.x86_64 90/140 2026-03-20T11:21:25.480 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : socat-1.7.4.1-8.el9.x86_64 91/140 2026-03-20T11:21:25.480 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 92/140 2026-03-20T11:21:25.480 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 93/140 2026-03-20T11:21:25.480 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : protobuf-compiler-3.14.0-17.el9.x86_64 94/140 2026-03-20T11:21:25.480 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : abseil-cpp-20211102.0-4.el9.x86_64 95/140 2026-03-20T11:21:25.480 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 96/140 2026-03-20T11:21:25.480 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 97/140 2026-03-20T11:21:25.480 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : libarrow-9.0.0-15.el9.x86_64 98/140 2026-03-20T11:21:25.480 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : libarrow-doc-9.0.0-15.el9.noarch 99/140 2026-03-20T11:21:25.480 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : liboath-2.6.12-1.el9.x86_64 100/140 2026-03-20T11:21:25.480 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 101/140 2026-03-20T11:21:25.480 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 102/140 2026-03-20T11:21:25.480 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : parquet-libs-9.0.0-15.el9.x86_64 103/140 2026-03-20T11:21:25.480 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 104/140 2026-03-20T11:21:25.480 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-autocommand-2.2.2-8.el9.noarch 105/140 2026-03-20T11:21:25.480 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-backports-tarfile-1.2.0-1.el9.noarch 106/140 2026-03-20T11:21:25.480 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 107/140 2026-03-20T11:21:25.480 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 108/140 2026-03-20T11:21:25.480 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 109/140 2026-03-20T11:21:25.480 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-cheroot-10.0.1-4.el9.noarch 110/140 2026-03-20T11:21:25.480 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 111/140 2026-03-20T11:21:25.480 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-google-auth-1:2.45.0-1.el9.noarch 112/140 2026-03-20T11:21:25.480 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 113/140 2026-03-20T11:21:25.480 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 114/140 2026-03-20T11:21:25.480 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 115/140 2026-03-20T11:21:25.480 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 116/140 2026-03-20T11:21:25.480 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 117/140 2026-03-20T11:21:25.480 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-jaraco-context-6.0.1-3.el9.noarch 118/140 2026-03-20T11:21:25.481 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 119/140 2026-03-20T11:21:25.481 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-jaraco-text-4.0.0-2.el9.noarch 120/140 2026-03-20T11:21:25.481 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-kubernetes-1:26.1.0-3.el9.noarch 121/140 2026-03-20T11:21:25.481 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 122/140 2026-03-20T11:21:25.481 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 123/140 2026-03-20T11:21:25.481 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 124/140 2026-03-20T11:21:25.481 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 125/140 2026-03-20T11:21:25.481 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 126/140 2026-03-20T11:21:25.481 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 127/140 2026-03-20T11:21:25.481 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 128/140 2026-03-20T11:21:25.481 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 129/140 2026-03-20T11:21:25.481 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-typing-extensions-4.15.0-1.el9.noarch 130/140 2026-03-20T11:21:25.481 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 131/140 2026-03-20T11:21:25.481 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-xmltodict-0.12.0-15.el9.noarch 132/140 2026-03-20T11:21:25.481 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 133/140 2026-03-20T11:21:25.481 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : re2-1:20211101-20.el9.x86_64 134/140 2026-03-20T11:21:25.481 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : s3cmd-2.4.0-1.el9.noarch 135/140 2026-03-20T11:21:25.481 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : thrift-0.15.0-4.el9.x86_64 136/140 2026-03-20T11:21:25.481 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : librados2-2:20.2.0-712.g70f8415b.el9.x86_64 137/140 2026-03-20T11:21:25.481 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 138/140 2026-03-20T11:21:25.481 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 139/140 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 140/140 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout:Upgraded: 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: librados2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout:Installed: 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: abseil-cpp-20211102.0-4.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: boost-program-options-1.75.0-13.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: bzip2-1.0.8-11.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: ceph-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: ceph-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: ceph-grafana-dashboards-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: ceph-immutable-object-cache-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: ceph-mgr-diskprediction-local-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: ceph-mgr-modules-core-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: ceph-prometheus-alerts-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: ceph-test-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: cephadm-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: cryptsetup-2.8.1-3.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: flexiblas-3.0.4-9.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: flexiblas-netlib-3.0.4-9.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: fuse-2.9.9-17.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: grpc-data-1.46.7-10.el9.noarch 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: ledmon-libs-1.1.0-3.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: libarrow-9.0.0-15.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: libarrow-doc-9.0.0-15.el9.noarch 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: libcephfs-devel-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: libconfig-1.7.2-9.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: libgfortran-11.5.0-14.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: libnbd-1.20.3-4.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: liboath-2.6.12-1.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: libpmemobj-1.12.1-1.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: libquadmath-11.5.0-14.el9.x86_64 2026-03-20T11:21:25.580 INFO:teuthology.orchestra.run.vm02.stdout: librabbitmq-0.11.0-7.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: librados-devel-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: librdkafka-1.6.1-102.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: libstoragemgmt-1.10.1-1.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: libunwind-1.6.2-1.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: libxslt-1.1.34-12.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: lttng-ust-2.12.0-6.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: lua-5.4.4-4.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: lua-devel-5.4.4-4.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: luarocks-3.9.2-5.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: mailcap-2.1.49-5.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: openblas-0.3.29-1.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: openblas-openmp-0.3.29-1.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: parquet-libs-9.0.0-15.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: pciutils-3.7.0-7.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: perl-Benchmark-1.23-483.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: perl-Test-Harness-1:3.42-461.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: protobuf-3.14.0-17.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: protobuf-compiler-3.14.0-17.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-autocommand-2.2.2-8.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-babel-2.9.1-2.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-backports-tarfile-1.2.0-1.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-cachetools-4.2.4-1.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-ceph-argparse-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-cephfs-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-certifi-2023.05.07-4.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-cffi-1.14.5-5.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-cheroot-10.0.1-4.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-cryptography-36.0.1-5.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-devel-3.9.25-3.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-google-auth-1:2.45.0-1.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-jaraco-8.2.1-3.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-jaraco-context-6.0.1-3.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-jaraco-text-4.0.0-2.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-jinja2-2.11.3-8.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-jmespath-1.0.1-1.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-kubernetes-1:26.1.0-3.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-libstoragemgmt-1.10.1-1.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-natsort-7.1.1-5.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-numpy-1:1.23.5-2.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-numpy-f2py-1:1.23.5-2.el9.x86_64 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-packaging-20.9-5.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-ply-3.11-14.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-portend-3.1.0-2.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-protobuf-3.14.0-17.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-pyasn1-0.4.8-7.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-pyasn1-modules-0.4.8-7.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-pycparser-2.20-6.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-pyparsing-2.4.7-9.el9.noarch 2026-03-20T11:21:25.581 INFO:teuthology.orchestra.run.vm02.stdout: python3-rados-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: python3-rbd-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: python3-repoze-lru-0.7-16.el9.noarch 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: python3-requests-2.25.1-10.el9.noarch 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: python3-rgw-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: python3-routes-2.5.1-5.el9.noarch 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: python3-rsa-4.9-2.el9.noarch 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: python3-scipy-1.9.3-2.el9.x86_64 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: python3-tempora-5.0.0-2.el9.noarch 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: python3-toml-0.10.2-6.el9.noarch 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: python3-typing-extensions-4.15.0-1.el9.noarch 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: python3-urllib3-1.26.5-7.el9.noarch 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: python3-xmltodict-0.12.0-15.el9.noarch 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: qatlib-25.08.0-2.el9.x86_64 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: qatlib-service-25.08.0-2.el9.x86_64 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: qatzip-libs-1.3.1-1.el9.x86_64 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: rbd-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: rbd-nbd-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: re2-1:20211101-20.el9.x86_64 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: s3cmd-2.4.0-1.el9.noarch 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: socat-1.7.4.1-8.el9.x86_64 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: thrift-0.15.0-4.el9.x86_64 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: unzip-6.0-59.el9.x86_64 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: zip-3.0-35.el9.x86_64 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:25.582 INFO:teuthology.orchestra.run.vm02.stdout:Complete! 2026-03-20T11:21:25.673 DEBUG:teuthology.parallel:result is None 2026-03-20T11:21:28.824 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-test-2:20.2.0-712.g70f8415b.el9.x86_64 132/140 2026-03-20T11:21:28.831 INFO:teuthology.orchestra.run.vm08.stdout: Installing : perl-Test-Harness-1:3.42-461.el9.noarch 133/140 2026-03-20T11:21:28.837 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libcephfs-devel-2:20.2.0-712.g70f8415b.el9.x86_6 134/140 2026-03-20T11:21:28.848 INFO:teuthology.orchestra.run.vm08.stdout: Installing : rbd-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 135/140 2026-03-20T11:21:28.866 INFO:teuthology.orchestra.run.vm08.stdout: Installing : rbd-nbd-2:20.2.0-712.g70f8415b.el9.x86_64 136/140 2026-03-20T11:21:28.873 INFO:teuthology.orchestra.run.vm08.stdout: Installing : s3cmd-2.4.0-1.el9.noarch 137/140 2026-03-20T11:21:28.877 INFO:teuthology.orchestra.run.vm08.stdout: Installing : bzip2-1.0.8-11.el9.x86_64 138/140 2026-03-20T11:21:28.877 INFO:teuthology.orchestra.run.vm08.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 139/140 2026-03-20T11:21:28.892 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 139/140 2026-03-20T11:21:28.892 INFO:teuthology.orchestra.run.vm08.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 140/140 2026-03-20T11:21:30.248 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 140/140 2026-03-20T11:21:30.248 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-2:20.2.0-712.g70f8415b.el9.x86_64 1/140 2026-03-20T11:21:30.248 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 2/140 2026-03-20T11:21:30.248 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 3/140 2026-03-20T11:21:30.248 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 4/140 2026-03-20T11:21:30.248 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-immutable-object-cache-2:20.2.0-712.g70f841 5/140 2026-03-20T11:21:30.248 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 6/140 2026-03-20T11:21:30.248 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 7/140 2026-03-20T11:21:30.248 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 8/140 2026-03-20T11:21:30.248 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 9/140 2026-03-20T11:21:30.248 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 10/140 2026-03-20T11:21:30.248 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 11/140 2026-03-20T11:21:30.248 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-test-2:20.2.0-712.g70f8415b.el9.x86_64 12/140 2026-03-20T11:21:30.248 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libcephfs-devel-2:20.2.0-712.g70f8415b.el9.x86_6 13/140 2026-03-20T11:21:30.248 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_ 14/140 2026-03-20T11:21:30.248 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 15/140 2026-03-20T11:21:30.249 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 16/140 2026-03-20T11:21:30.249 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : librados-devel-2:20.2.0-712.g70f8415b.el9.x86_64 17/140 2026-03-20T11:21:30.249 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_ 18/140 2026-03-20T11:21:30.249 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 19/140 2026-03-20T11:21:30.249 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-ceph-argparse-2:20.2.0-712.g70f8415b.el9 20/140 2026-03-20T11:21:30.249 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-ceph-common-2:20.2.0-712.g70f8415b.el9.x 21/140 2026-03-20T11:21:30.249 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-cephfs-2:20.2.0-712.g70f8415b.el9.x86_64 22/140 2026-03-20T11:21:30.249 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-rados-2:20.2.0-712.g70f8415b.el9.x86_64 23/140 2026-03-20T11:21:30.249 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-rbd-2:20.2.0-712.g70f8415b.el9.x86_64 24/140 2026-03-20T11:21:30.249 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-rgw-2:20.2.0-712.g70f8415b.el9.x86_64 25/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : rbd-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 26/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 27/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : rbd-nbd-2:20.2.0-712.g70f8415b.el9.x86_64 28/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-grafana-dashboards-2:20.2.0-712.g70f8415b.e 29/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noar 30/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.no 31/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-mgr-diskprediction-local-2:20.2.0-712.g70f8 32/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-mgr-modules-core-2:20.2.0-712.g70f8415b.el9 33/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 34/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-prometheus-alerts-2:20.2.0-712.g70f8415b.el 35/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 36/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : cephadm-2:20.2.0-712.g70f8415b.el9.noarch 37/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : bzip2-1.0.8-11.el9.x86_64 38/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : cryptsetup-2.8.1-3.el9.x86_64 39/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : fuse-2.9.9-17.el9.x86_64 40/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ledmon-libs-1.1.0-3.el9.x86_64 41/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libconfig-1.7.2-9.el9.x86_64 42/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libgfortran-11.5.0-14.el9.x86_64 43/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libquadmath-11.5.0-14.el9.x86_64 44/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : mailcap-2.1.49-5.el9.noarch 45/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : pciutils-3.7.0-7.el9.x86_64 46/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 47/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-cryptography-36.0.1-5.el9.x86_64 48/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-ply-3.11-14.el9.noarch 49/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 50/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-pyparsing-2.4.7-9.el9.noarch 51/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-requests-2.25.1-10.el9.noarch 52/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-urllib3-1.26.5-7.el9.noarch 53/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : unzip-6.0-59.el9.x86_64 54/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : zip-3.0-35.el9.x86_64 55/140 2026-03-20T11:21:30.250 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : boost-program-options-1.75.0-13.el9.x86_64 56/140 2026-03-20T11:21:30.251 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : flexiblas-3.0.4-9.el9.x86_64 57/140 2026-03-20T11:21:30.251 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : flexiblas-netlib-3.0.4-9.el9.x86_64 58/140 2026-03-20T11:21:30.251 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 59/140 2026-03-20T11:21:30.251 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libnbd-1.20.3-4.el9.x86_64 60/140 2026-03-20T11:21:30.251 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 61/140 2026-03-20T11:21:30.251 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 62/140 2026-03-20T11:21:30.251 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 63/140 2026-03-20T11:21:30.251 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libstoragemgmt-1.10.1-1.el9.x86_64 64/140 2026-03-20T11:21:30.251 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libxslt-1.1.34-12.el9.x86_64 65/140 2026-03-20T11:21:30.251 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 66/140 2026-03-20T11:21:30.251 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : lua-5.4.4-4.el9.x86_64 67/140 2026-03-20T11:21:30.251 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : openblas-0.3.29-1.el9.x86_64 68/140 2026-03-20T11:21:30.251 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : openblas-openmp-0.3.29-1.el9.x86_64 69/140 2026-03-20T11:21:30.251 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : perl-Benchmark-1.23-483.el9.noarch 70/140 2026-03-20T11:21:30.251 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : perl-Test-Harness-1:3.42-461.el9.noarch 71/140 2026-03-20T11:21:30.251 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : protobuf-3.14.0-17.el9.x86_64 72/140 2026-03-20T11:21:30.251 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 73/140 2026-03-20T11:21:30.251 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-devel-3.9.25-3.el9.x86_64 74/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-jinja2-2.11.3-8.el9.noarch 75/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-jmespath-1.0.1-1.el9.noarch 76/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-libstoragemgmt-1.10.1-1.el9.x86_64 77/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 78/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-numpy-1:1.23.5-2.el9.x86_64 79/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 80/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 81/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-protobuf-3.14.0-17.el9.noarch 82/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-pyasn1-0.4.8-7.el9.noarch 83/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-pyasn1-modules-0.4.8-7.el9.noarch 84/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 85/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-scipy-1.9.3-2.el9.x86_64 86/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 87/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : qatlib-25.08.0-2.el9.x86_64 88/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : qatlib-service-25.08.0-2.el9.x86_64 89/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : qatzip-libs-1.3.1-1.el9.x86_64 90/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : socat-1.7.4.1-8.el9.x86_64 91/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 92/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 93/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : protobuf-compiler-3.14.0-17.el9.x86_64 94/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : abseil-cpp-20211102.0-4.el9.x86_64 95/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 96/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 97/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libarrow-9.0.0-15.el9.x86_64 98/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libarrow-doc-9.0.0-15.el9.noarch 99/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : liboath-2.6.12-1.el9.x86_64 100/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 101/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 102/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : parquet-libs-9.0.0-15.el9.x86_64 103/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 104/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-autocommand-2.2.2-8.el9.noarch 105/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-backports-tarfile-1.2.0-1.el9.noarch 106/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 107/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 108/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 109/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-cheroot-10.0.1-4.el9.noarch 110/140 2026-03-20T11:21:30.252 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 111/140 2026-03-20T11:21:30.253 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-google-auth-1:2.45.0-1.el9.noarch 112/140 2026-03-20T11:21:30.253 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 113/140 2026-03-20T11:21:30.253 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 114/140 2026-03-20T11:21:30.253 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 115/140 2026-03-20T11:21:30.253 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 116/140 2026-03-20T11:21:30.253 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 117/140 2026-03-20T11:21:30.253 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-jaraco-context-6.0.1-3.el9.noarch 118/140 2026-03-20T11:21:30.253 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 119/140 2026-03-20T11:21:30.253 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-jaraco-text-4.0.0-2.el9.noarch 120/140 2026-03-20T11:21:30.253 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-kubernetes-1:26.1.0-3.el9.noarch 121/140 2026-03-20T11:21:30.253 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 122/140 2026-03-20T11:21:30.253 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 123/140 2026-03-20T11:21:30.253 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 124/140 2026-03-20T11:21:30.253 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 125/140 2026-03-20T11:21:30.254 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 126/140 2026-03-20T11:21:30.254 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 127/140 2026-03-20T11:21:30.254 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 128/140 2026-03-20T11:21:30.254 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 129/140 2026-03-20T11:21:30.254 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-typing-extensions-4.15.0-1.el9.noarch 130/140 2026-03-20T11:21:30.254 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 131/140 2026-03-20T11:21:30.254 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-xmltodict-0.12.0-15.el9.noarch 132/140 2026-03-20T11:21:30.254 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 133/140 2026-03-20T11:21:30.254 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : re2-1:20211101-20.el9.x86_64 134/140 2026-03-20T11:21:30.254 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : s3cmd-2.4.0-1.el9.noarch 135/140 2026-03-20T11:21:30.254 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : thrift-0.15.0-4.el9.x86_64 136/140 2026-03-20T11:21:30.254 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : librados2-2:20.2.0-712.g70f8415b.el9.x86_64 137/140 2026-03-20T11:21:30.254 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 138/140 2026-03-20T11:21:30.254 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 139/140 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 140/140 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout:Upgraded: 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: librados2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout:Installed: 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: abseil-cpp-20211102.0-4.el9.x86_64 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: boost-program-options-1.75.0-13.el9.x86_64 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: bzip2-1.0.8-11.el9.x86_64 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: ceph-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: ceph-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: ceph-grafana-dashboards-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: ceph-immutable-object-cache-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-diskprediction-local-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-modules-core-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: ceph-prometheus-alerts-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: ceph-test-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: cephadm-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: cryptsetup-2.8.1-3.el9.x86_64 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: flexiblas-3.0.4-9.el9.x86_64 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: flexiblas-netlib-3.0.4-9.el9.x86_64 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: fuse-2.9.9-17.el9.x86_64 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: grpc-data-1.46.7-10.el9.noarch 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: ledmon-libs-1.1.0-3.el9.x86_64 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: libarrow-9.0.0-15.el9.x86_64 2026-03-20T11:21:30.347 INFO:teuthology.orchestra.run.vm08.stdout: libarrow-doc-9.0.0-15.el9.noarch 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: libcephfs-devel-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: libconfig-1.7.2-9.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: libgfortran-11.5.0-14.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: libnbd-1.20.3-4.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: liboath-2.6.12-1.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: libpmemobj-1.12.1-1.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: libquadmath-11.5.0-14.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: librabbitmq-0.11.0-7.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: librados-devel-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: librdkafka-1.6.1-102.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: libstoragemgmt-1.10.1-1.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: libunwind-1.6.2-1.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: libxslt-1.1.34-12.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: lttng-ust-2.12.0-6.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: lua-5.4.4-4.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: lua-devel-5.4.4-4.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: luarocks-3.9.2-5.el9.noarch 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: mailcap-2.1.49-5.el9.noarch 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: openblas-0.3.29-1.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: openblas-openmp-0.3.29-1.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: parquet-libs-9.0.0-15.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: pciutils-3.7.0-7.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: perl-Benchmark-1.23-483.el9.noarch 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: perl-Test-Harness-1:3.42-461.el9.noarch 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: protobuf-3.14.0-17.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: protobuf-compiler-3.14.0-17.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-autocommand-2.2.2-8.el9.noarch 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-babel-2.9.1-2.el9.noarch 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-backports-tarfile-1.2.0-1.el9.noarch 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-cachetools-4.2.4-1.el9.noarch 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-ceph-argparse-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-cephfs-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-certifi-2023.05.07-4.el9.noarch 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-cffi-1.14.5-5.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-cheroot-10.0.1-4.el9.noarch 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-cryptography-36.0.1-5.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-devel-3.9.25-3.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-google-auth-1:2.45.0-1.el9.noarch 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-8.2.1-3.el9.noarch 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-context-6.0.1-3.el9.noarch 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-text-4.0.0-2.el9.noarch 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-jinja2-2.11.3-8.el9.noarch 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-jmespath-1.0.1-1.el9.noarch 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-kubernetes-1:26.1.0-3.el9.noarch 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-libstoragemgmt-1.10.1-1.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-natsort-7.1.1-5.el9.noarch 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-numpy-1:1.23.5-2.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-numpy-f2py-1:1.23.5-2.el9.x86_64 2026-03-20T11:21:30.348 INFO:teuthology.orchestra.run.vm08.stdout: python3-packaging-20.9-5.el9.noarch 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: python3-ply-3.11-14.el9.noarch 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: python3-portend-3.1.0-2.el9.noarch 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: python3-protobuf-3.14.0-17.el9.noarch 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: python3-pyasn1-0.4.8-7.el9.noarch 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: python3-pyasn1-modules-0.4.8-7.el9.noarch 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: python3-pycparser-2.20-6.el9.noarch 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: python3-pyparsing-2.4.7-9.el9.noarch 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: python3-rados-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: python3-rbd-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: python3-repoze-lru-0.7-16.el9.noarch 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: python3-requests-2.25.1-10.el9.noarch 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: python3-rgw-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: python3-routes-2.5.1-5.el9.noarch 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: python3-rsa-4.9-2.el9.noarch 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: python3-scipy-1.9.3-2.el9.x86_64 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: python3-tempora-5.0.0-2.el9.noarch 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: python3-toml-0.10.2-6.el9.noarch 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: python3-typing-extensions-4.15.0-1.el9.noarch 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: python3-urllib3-1.26.5-7.el9.noarch 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: python3-xmltodict-0.12.0-15.el9.noarch 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: qatlib-25.08.0-2.el9.x86_64 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: qatlib-service-25.08.0-2.el9.x86_64 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: qatzip-libs-1.3.1-1.el9.x86_64 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: rbd-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: rbd-nbd-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: re2-1:20211101-20.el9.x86_64 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: s3cmd-2.4.0-1.el9.noarch 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: socat-1.7.4.1-8.el9.x86_64 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: thrift-0.15.0-4.el9.x86_64 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: unzip-6.0-59.el9.x86_64 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: zip-3.0-35.el9.x86_64 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:30.349 INFO:teuthology.orchestra.run.vm08.stdout:Complete! 2026-03-20T11:21:30.440 DEBUG:teuthology.parallel:result is None 2026-03-20T11:21:30.440 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=70f8415b300f041766fa27faf7d5472699e32388 2026-03-20T11:21:31.095 DEBUG:teuthology.orchestra.run.vm02:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2026-03-20T11:21:31.113 INFO:teuthology.orchestra.run.vm02.stdout:20.2.0-712.g70f8415b.el9 2026-03-20T11:21:31.113 INFO:teuthology.packaging:The installed version of ceph is 20.2.0-712.g70f8415b.el9 2026-03-20T11:21:31.113 INFO:teuthology.task.install:The correct ceph version 20.2.0-712.g70f8415b is installed. 2026-03-20T11:21:31.114 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=70f8415b300f041766fa27faf7d5472699e32388 2026-03-20T11:21:31.717 DEBUG:teuthology.orchestra.run.vm05:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2026-03-20T11:21:31.736 INFO:teuthology.orchestra.run.vm05.stdout:20.2.0-712.g70f8415b.el9 2026-03-20T11:21:31.736 INFO:teuthology.packaging:The installed version of ceph is 20.2.0-712.g70f8415b.el9 2026-03-20T11:21:31.736 INFO:teuthology.task.install:The correct ceph version 20.2.0-712.g70f8415b is installed. 2026-03-20T11:21:31.737 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=70f8415b300f041766fa27faf7d5472699e32388 2026-03-20T11:21:32.313 DEBUG:teuthology.orchestra.run.vm07:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2026-03-20T11:21:32.333 INFO:teuthology.orchestra.run.vm07.stdout:20.2.0-712.g70f8415b.el9 2026-03-20T11:21:32.333 INFO:teuthology.packaging:The installed version of ceph is 20.2.0-712.g70f8415b.el9 2026-03-20T11:21:32.333 INFO:teuthology.task.install:The correct ceph version 20.2.0-712.g70f8415b is installed. 2026-03-20T11:21:32.334 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=70f8415b300f041766fa27faf7d5472699e32388 2026-03-20T11:21:32.930 DEBUG:teuthology.orchestra.run.vm08:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2026-03-20T11:21:32.948 INFO:teuthology.orchestra.run.vm08.stdout:20.2.0-712.g70f8415b.el9 2026-03-20T11:21:32.948 INFO:teuthology.packaging:The installed version of ceph is 20.2.0-712.g70f8415b.el9 2026-03-20T11:21:32.948 INFO:teuthology.task.install:The correct ceph version 20.2.0-712.g70f8415b is installed. 2026-03-20T11:21:32.949 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2026-03-20T11:21:32.949 DEBUG:teuthology.orchestra.run.vm02:> set -ex 2026-03-20T11:21:32.949 DEBUG:teuthology.orchestra.run.vm02:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-03-20T11:21:32.973 DEBUG:teuthology.orchestra.run.vm05:> set -ex 2026-03-20T11:21:32.973 DEBUG:teuthology.orchestra.run.vm05:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-03-20T11:21:32.999 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-20T11:21:32.999 DEBUG:teuthology.orchestra.run.vm07:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-03-20T11:21:33.024 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-20T11:21:33.024 DEBUG:teuthology.orchestra.run.vm08:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-03-20T11:21:33.048 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2026-03-20T11:21:33.048 DEBUG:teuthology.orchestra.run.vm02:> set -ex 2026-03-20T11:21:33.048 DEBUG:teuthology.orchestra.run.vm02:> sudo dd of=/usr/bin/daemon-helper 2026-03-20T11:21:33.069 DEBUG:teuthology.orchestra.run.vm02:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-03-20T11:21:33.130 DEBUG:teuthology.orchestra.run.vm05:> set -ex 2026-03-20T11:21:33.130 DEBUG:teuthology.orchestra.run.vm05:> sudo dd of=/usr/bin/daemon-helper 2026-03-20T11:21:33.153 DEBUG:teuthology.orchestra.run.vm05:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-03-20T11:21:33.216 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-20T11:21:33.216 DEBUG:teuthology.orchestra.run.vm07:> sudo dd of=/usr/bin/daemon-helper 2026-03-20T11:21:33.240 DEBUG:teuthology.orchestra.run.vm07:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-03-20T11:21:33.303 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-20T11:21:33.303 DEBUG:teuthology.orchestra.run.vm08:> sudo dd of=/usr/bin/daemon-helper 2026-03-20T11:21:33.325 DEBUG:teuthology.orchestra.run.vm08:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-03-20T11:21:33.386 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2026-03-20T11:21:33.386 DEBUG:teuthology.orchestra.run.vm02:> set -ex 2026-03-20T11:21:33.386 DEBUG:teuthology.orchestra.run.vm02:> sudo dd of=/usr/bin/adjust-ulimits 2026-03-20T11:21:33.407 DEBUG:teuthology.orchestra.run.vm02:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-03-20T11:21:33.469 DEBUG:teuthology.orchestra.run.vm05:> set -ex 2026-03-20T11:21:33.469 DEBUG:teuthology.orchestra.run.vm05:> sudo dd of=/usr/bin/adjust-ulimits 2026-03-20T11:21:33.492 DEBUG:teuthology.orchestra.run.vm05:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-03-20T11:21:33.555 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-20T11:21:33.555 DEBUG:teuthology.orchestra.run.vm07:> sudo dd of=/usr/bin/adjust-ulimits 2026-03-20T11:21:33.579 DEBUG:teuthology.orchestra.run.vm07:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-03-20T11:21:33.643 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-20T11:21:33.643 DEBUG:teuthology.orchestra.run.vm08:> sudo dd of=/usr/bin/adjust-ulimits 2026-03-20T11:21:33.666 DEBUG:teuthology.orchestra.run.vm08:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-03-20T11:21:33.726 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2026-03-20T11:21:33.727 DEBUG:teuthology.orchestra.run.vm02:> set -ex 2026-03-20T11:21:33.727 DEBUG:teuthology.orchestra.run.vm02:> sudo dd of=/usr/bin/stdin-killer 2026-03-20T11:21:33.748 DEBUG:teuthology.orchestra.run.vm02:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-03-20T11:21:33.810 DEBUG:teuthology.orchestra.run.vm05:> set -ex 2026-03-20T11:21:33.810 DEBUG:teuthology.orchestra.run.vm05:> sudo dd of=/usr/bin/stdin-killer 2026-03-20T11:21:33.834 DEBUG:teuthology.orchestra.run.vm05:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-03-20T11:21:33.898 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-20T11:21:33.898 DEBUG:teuthology.orchestra.run.vm07:> sudo dd of=/usr/bin/stdin-killer 2026-03-20T11:21:33.922 DEBUG:teuthology.orchestra.run.vm07:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-03-20T11:21:33.986 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-20T11:21:33.986 DEBUG:teuthology.orchestra.run.vm08:> sudo dd of=/usr/bin/stdin-killer 2026-03-20T11:21:34.007 DEBUG:teuthology.orchestra.run.vm08:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-03-20T11:21:34.069 INFO:teuthology.run_tasks:Running task ceph... 2026-03-20T11:21:34.109 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2026-03-20T11:21:34.109 DEBUG:teuthology.orchestra.run.vm02:> sudo chmod 777 /var/log/ceph 2026-03-20T11:21:34.110 DEBUG:teuthology.orchestra.run.vm05:> sudo chmod 777 /var/log/ceph 2026-03-20T11:21:34.112 DEBUG:teuthology.orchestra.run.vm07:> sudo chmod 777 /var/log/ceph 2026-03-20T11:21:34.114 DEBUG:teuthology.orchestra.run.vm08:> sudo chmod 777 /var/log/ceph 2026-03-20T11:21:34.137 INFO:tasks.ceph:Disabling ceph logrotate... 2026-03-20T11:21:34.138 DEBUG:teuthology.orchestra.run.vm02:> sudo rm -f -- /etc/logrotate.d/ceph 2026-03-20T11:21:34.178 DEBUG:teuthology.orchestra.run.vm05:> sudo rm -f -- /etc/logrotate.d/ceph 2026-03-20T11:21:34.179 DEBUG:teuthology.orchestra.run.vm07:> sudo rm -f -- /etc/logrotate.d/ceph 2026-03-20T11:21:34.181 DEBUG:teuthology.orchestra.run.vm08:> sudo rm -f -- /etc/logrotate.d/ceph 2026-03-20T11:21:34.205 INFO:tasks.ceph:Creating extra log directories... 2026-03-20T11:21:34.205 DEBUG:teuthology.orchestra.run.vm02:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-03-20T11:21:34.244 DEBUG:teuthology.orchestra.run.vm05:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-03-20T11:21:34.246 DEBUG:teuthology.orchestra.run.vm07:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-03-20T11:21:34.248 DEBUG:teuthology.orchestra.run.vm08:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-03-20T11:21:34.278 INFO:tasks.ceph:Creating ceph cluster c1... 2026-03-20T11:21:34.278 INFO:tasks.ceph:config {'conf': {'client': {'debug rgw': 20, 'debug rgw lifecycle': 20, 'debug rgw notification': 20, 'debug rgw sync': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=', 'rgw curl low speed time': 300, 'rgw data log num shards': 4, 'rgw data sync poll interval': 5, 'rgw md log max shards': 4, 'rgw meta sync poll interval': 5, 'rgw s3 auth use sts': True, 'rgw sts key': 'abcdefghijklmnoq', 'rgw sync data inject err probability': 0, 'rgw sync log trim interval': 0, 'rgw sync meta inject err probability': 0, 'rgw sync obj etag verify': True, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'client.0': {'rgw data notify interval msec': 0}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd mclock iops capacity threshold hdd': 49000, 'rocksdb delete range threshold': 0}, 'osd.0': {'osd_max_omap_entries_per_request': 10}, 'osd.1': {'osd_max_omap_entries_per_request': 1000}, 'osd.2': {'osd_max_omap_entries_per_request': 10000}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(PG_AVAILABILITY\\)', '\\(PG_DEGRADED\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'not have an application enabled'], 'cpu_profile': set(), 'cluster': 'c1', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-03-20T11:21:34.278 INFO:tasks.ceph:ctx.config {'archive_path': '/archive/kyr-2026-03-20_10:58:43-rgw-tentacle-none-default-vps/2068', 'branch': 'tentacle', 'description': 'rgw/multisite/{clusters frontend/beast ignore-pg-availability notify omap_limits overrides realms/two-zonegroup supported-random-distro$/{centos_latest} tasks/test_multi}', 'email': None, 'first_in_suite': False, 'flavor': 'default', 'job_id': '2068', 'last_in_suite': False, 'machine_type': 'vps', 'name': 'kyr-2026-03-20_10:58:43-rgw-tentacle-none-default-vps', 'no_nested_subset': False, 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'tentacle'}, 'ansible.cephlab': {'branch': 'main', 'repo': 'https://github.com/kshtsk/ceph-cm-ansible.git', 'skip_tags': 'nagios,monitoring-scripts,hostname,pubkeys,zap,sudoers,kerberos,ntp-client,resolvconf,cpan,nfs', 'vars': {'logical_volumes': {'lv_1': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}, 'lv_2': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}, 'lv_3': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}, 'lv_4': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}}, 'timezone': 'UTC', 'volume_groups': {'vg_nvme': {'pvs': '/dev/vdb,/dev/vdc,/dev/vdd,/dev/vde'}}}}, 'ceph': {'conf': {'client': {'debug rgw': 20, 'debug rgw lifecycle': 20, 'debug rgw notification': 20, 'debug rgw sync': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=', 'rgw curl low speed time': 300, 'rgw data log num shards': 4, 'rgw data sync poll interval': 5, 'rgw md log max shards': 4, 'rgw meta sync poll interval': 5, 'rgw s3 auth use sts': True, 'rgw sts key': 'abcdefghijklmnoq', 'rgw sync data inject err probability': 0, 'rgw sync log trim interval': 0, 'rgw sync meta inject err probability': 0, 'rgw sync obj etag verify': True, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'client.0': {'rgw data notify interval msec': 0}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd mclock iops capacity threshold hdd': 49000, 'rocksdb delete range threshold': 0}, 'osd.0': {'osd_max_omap_entries_per_request': 10}, 'osd.1': {'osd_max_omap_entries_per_request': 1000}, 'osd.2': {'osd_max_omap_entries_per_request': 10000}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(PG_AVAILABILITY\\)', '\\(PG_DEGRADED\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'not have an application enabled'], 'sha1': '70f8415b300f041766fa27faf7d5472699e32388'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'cephadm': {'cephadm_binary_url': 'https://download.ceph.com/rpm-20.2.0/el9/noarch/cephadm'}, 'install': {'ceph': {'flavor': 'default', 'sha1': '70f8415b300f041766fa27faf7d5472699e32388'}, 'extra_system_packages': {'deb': ['python3-jmespath', 'python3-xmltodict', 's3cmd'], 'rpm': ['bzip2', 'perl-Test-Harness', 'python3-jmespath', 'python3-xmltodict', 's3cmd']}}, 'rgw': {'compression type': 'random', 'frontend': 'beast'}, 'rgw-multisite': {'realm': {'is_default': True, 'name': 'test-realm'}, 'zonegroups': [{'enabled_features': ['resharding', 'notification_v2'], 'endpoints': ['c1.client.0'], 'is_default': True, 'is_master': True, 'name': 'a', 'zones': [{'endpoints': ['c1.client.0'], 'is_default': True, 'is_master': True, 'name': 'a1'}, {'endpoints': ['c1.client.1'], 'name': 'a2'}]}, {'enabled_features': ['resharding', 'notification_v2'], 'endpoints': ['c2.client.0'], 'is_default': True, 'name': 'b', 'zones': [{'endpoints': ['c2.client.0'], 'is_default': True, 'is_master': True, 'name': 'b1'}, {'endpoints': ['c2.client.1'], 'name': 'b2'}]}]}, 'rgw-multisite-tests': {'args': ['tests.py', '-a', '!fails_with_rgw']}, 'workunit': {'branch': 'tt-tentacle', 'sha1': '7b4fb1902b22ff4ea3f4ff6a953bc42198ebeffe'}}, 'owner': 'kyr', 'priority': 1000, 'repo': 'https://github.com/ceph/ceph.git', 'roles': [['c1.mon.a', 'c1.osd.0', 'c1.osd.1', 'c1.osd.2', 'c1.client.0'], ['c1.mgr.x', 'c1.osd.3', 'c1.osd.4', 'c1.osd.5', 'c1.client.1'], ['c2.mon.a', 'c2.osd.0', 'c2.osd.1', 'c2.osd.2', 'c2.client.0'], ['c2.mgr.x', 'c2.osd.3', 'c2.osd.4', 'c2.osd.5', 'c2.client.1']], 'seed': 7702, 'sha1': '70f8415b300f041766fa27faf7d5472699e32388', 'sleep_before_teardown': 0, 'suite': 'rgw', 'suite_branch': 'tt-tentacle', 'suite_path': '/home/teuthos/src/github.com_kshtsk_ceph_7b4fb1902b22ff4ea3f4ff6a953bc42198ebeffe/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/kshtsk/ceph.git', 'suite_sha1': '7b4fb1902b22ff4ea3f4ff6a953bc42198ebeffe', 'targets': {'vm02.local': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHDOdGzmlLQ/ZTYxKoEn2ygRbjgX4ux/lNWXbj7ocH2unOqJgDf8+OP1BWuHYNEv7h7psNGXZ0ZkhElY2KRateg=', 'vm05.local': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIs+bAjIBCO2Z1GDSS25TON+TMyqw5NQcnquTRycytbaJNkFJi5yDkOkkdeCgrmVAOljPOfw/yGvTfVtaKsCVhw=', 'vm07.local': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDlMCt1zzzTlX2IlgevpvFzoeoxOiYD4Qk8mFXF1w3snFJ7povvBC+RimEI07rEIHxU3hlLnvg+Tit+Yf5M5DAs=', 'vm08.local': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNug/VtQYdbJ4Dz5jL/XJ5LvMMhWwcvBq6Amz05ZwPG64TWDh0niEw8yvtwAlDN8ZBITlJN1keEDVaud9T22nD4='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': None}, {'ceph': {'cluster': 'c1', 'conf': {'client': {'debug rgw': 20, 'debug rgw lifecycle': 20, 'debug rgw notification': 20, 'debug rgw sync': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=', 'rgw curl low speed time': 300, 'rgw data log num shards': 4, 'rgw data sync poll interval': 5, 'rgw md log max shards': 4, 'rgw meta sync poll interval': 5, 'rgw s3 auth use sts': True, 'rgw sts key': 'abcdefghijklmnoq', 'rgw sync data inject err probability': 0, 'rgw sync log trim interval': 0, 'rgw sync meta inject err probability': 0, 'rgw sync obj etag verify': True, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'client.0': {'rgw data notify interval msec': 0}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd mclock iops capacity threshold hdd': 49000, 'rocksdb delete range threshold': 0}, 'osd.0': {'osd_max_omap_entries_per_request': 10}, 'osd.1': {'osd_max_omap_entries_per_request': 1000}, 'osd.2': {'osd_max_omap_entries_per_request': 10000}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(PG_AVAILABILITY\\)', '\\(PG_DEGRADED\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'not have an application enabled'], 'sha1': '70f8415b300f041766fa27faf7d5472699e32388'}}, {'ceph': {'cluster': 'c2'}}, {'rgw': {'c1.client.0': {'port': 8000}, 'c1.client.1': {'port': 8001}, 'c2.client.0': {'port': 8000}, 'c2.client.1': {'port': 8001}}}, {'rgw-multisite': None}, {'rgw-multisite-tests': {'config': {'reconfigure_delay': 90}}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'clyso-debian-13', 'teuthology_repo': 'https://github.com/clyso/teuthology', 'teuthology_sha1': '1c580df7a9c7c2aadc272da296344fd99f27c444', 'timestamp': '2026-03-20_10:58:43', 'tube': 'vps', 'user': 'kyr', 'verbose': False, 'worker_log': '/home/teuthos/.teuthology/dispatcher/dispatcher.vps.4188345'} 2026-03-20T11:21:34.278 DEBUG:teuthology.orchestra.run.vm02:> install -d -m0755 -- /home/ubuntu/cephtest/c1.data 2026-03-20T11:21:34.312 DEBUG:teuthology.orchestra.run.vm05:> install -d -m0755 -- /home/ubuntu/cephtest/c1.data 2026-03-20T11:21:34.313 DEBUG:teuthology.orchestra.run.vm07:> install -d -m0755 -- /home/ubuntu/cephtest/c1.data 2026-03-20T11:21:34.316 DEBUG:teuthology.orchestra.run.vm08:> install -d -m0755 -- /home/ubuntu/cephtest/c1.data 2026-03-20T11:21:34.333 DEBUG:teuthology.orchestra.run.vm02:> sudo install -d -m0777 -- /var/run/ceph 2026-03-20T11:21:34.368 DEBUG:teuthology.orchestra.run.vm05:> sudo install -d -m0777 -- /var/run/ceph 2026-03-20T11:21:34.371 DEBUG:teuthology.orchestra.run.vm07:> sudo install -d -m0777 -- /var/run/ceph 2026-03-20T11:21:34.373 DEBUG:teuthology.orchestra.run.vm08:> sudo install -d -m0777 -- /var/run/ceph 2026-03-20T11:21:34.399 DEBUG:teuthology.orchestra.run.vm02:> set -ex 2026-03-20T11:21:34.399 DEBUG:teuthology.orchestra.run.vm02:> dd if=/scratch_devs of=/dev/stdout 2026-03-20T11:21:34.445 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-03-20T11:21:34.445 DEBUG:teuthology.orchestra.run.vm02:> stat /dev/vg_nvme/lv_1 2026-03-20T11:21:34.501 INFO:teuthology.orchestra.run.vm02.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-03-20T11:21:34.501 INFO:teuthology.orchestra.run.vm02.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-20T11:21:34.501 INFO:teuthology.orchestra.run.vm02.stdout:Device: 6h/6d Inode: 631 Links: 1 2026-03-20T11:21:34.501 INFO:teuthology.orchestra.run.vm02.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-20T11:21:34.501 INFO:teuthology.orchestra.run.vm02.stdout:Context: system_u:object_r:device_t:s0 2026-03-20T11:21:34.501 INFO:teuthology.orchestra.run.vm02.stdout:Access: 2026-03-20 11:21:24.342445458 +0000 2026-03-20T11:21:34.501 INFO:teuthology.orchestra.run.vm02.stdout:Modify: 2026-03-20 11:19:56.082933672 +0000 2026-03-20T11:21:34.501 INFO:teuthology.orchestra.run.vm02.stdout:Change: 2026-03-20 11:19:56.082933672 +0000 2026-03-20T11:21:34.501 INFO:teuthology.orchestra.run.vm02.stdout: Birth: 2026-03-20 11:19:56.082933672 +0000 2026-03-20T11:21:34.501 DEBUG:teuthology.orchestra.run.vm02:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-03-20T11:21:34.561 INFO:teuthology.orchestra.run.vm02.stderr:1+0 records in 2026-03-20T11:21:34.561 INFO:teuthology.orchestra.run.vm02.stderr:1+0 records out 2026-03-20T11:21:34.561 INFO:teuthology.orchestra.run.vm02.stderr:512 bytes copied, 0.000239078 s, 2.1 MB/s 2026-03-20T11:21:34.562 DEBUG:teuthology.orchestra.run.vm02:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-03-20T11:21:34.615 DEBUG:teuthology.orchestra.run.vm02:> stat /dev/vg_nvme/lv_2 2026-03-20T11:21:34.670 INFO:teuthology.orchestra.run.vm02.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-03-20T11:21:34.670 INFO:teuthology.orchestra.run.vm02.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-20T11:21:34.670 INFO:teuthology.orchestra.run.vm02.stdout:Device: 6h/6d Inode: 687 Links: 1 2026-03-20T11:21:34.670 INFO:teuthology.orchestra.run.vm02.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-20T11:21:34.670 INFO:teuthology.orchestra.run.vm02.stdout:Context: system_u:object_r:device_t:s0 2026-03-20T11:21:34.670 INFO:teuthology.orchestra.run.vm02.stdout:Access: 2026-03-20 11:21:24.342445458 +0000 2026-03-20T11:21:34.670 INFO:teuthology.orchestra.run.vm02.stdout:Modify: 2026-03-20 11:19:56.327934413 +0000 2026-03-20T11:21:34.670 INFO:teuthology.orchestra.run.vm02.stdout:Change: 2026-03-20 11:19:56.327934413 +0000 2026-03-20T11:21:34.670 INFO:teuthology.orchestra.run.vm02.stdout: Birth: 2026-03-20 11:19:56.327934413 +0000 2026-03-20T11:21:34.670 DEBUG:teuthology.orchestra.run.vm02:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-03-20T11:21:34.730 INFO:teuthology.orchestra.run.vm02.stderr:1+0 records in 2026-03-20T11:21:34.730 INFO:teuthology.orchestra.run.vm02.stderr:1+0 records out 2026-03-20T11:21:34.730 INFO:teuthology.orchestra.run.vm02.stderr:512 bytes copied, 0.000175769 s, 2.9 MB/s 2026-03-20T11:21:34.731 DEBUG:teuthology.orchestra.run.vm02:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-03-20T11:21:34.785 DEBUG:teuthology.orchestra.run.vm02:> stat /dev/vg_nvme/lv_3 2026-03-20T11:21:34.840 INFO:teuthology.orchestra.run.vm02.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-03-20T11:21:34.840 INFO:teuthology.orchestra.run.vm02.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-20T11:21:34.840 INFO:teuthology.orchestra.run.vm02.stdout:Device: 6h/6d Inode: 725 Links: 1 2026-03-20T11:21:34.840 INFO:teuthology.orchestra.run.vm02.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-20T11:21:34.841 INFO:teuthology.orchestra.run.vm02.stdout:Context: system_u:object_r:device_t:s0 2026-03-20T11:21:34.841 INFO:teuthology.orchestra.run.vm02.stdout:Access: 2026-03-20 11:21:24.342445458 +0000 2026-03-20T11:21:34.841 INFO:teuthology.orchestra.run.vm02.stdout:Modify: 2026-03-20 11:19:56.607935260 +0000 2026-03-20T11:21:34.841 INFO:teuthology.orchestra.run.vm02.stdout:Change: 2026-03-20 11:19:56.607935260 +0000 2026-03-20T11:21:34.841 INFO:teuthology.orchestra.run.vm02.stdout: Birth: 2026-03-20 11:19:56.607935260 +0000 2026-03-20T11:21:34.841 DEBUG:teuthology.orchestra.run.vm02:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-03-20T11:21:34.903 INFO:teuthology.orchestra.run.vm02.stderr:1+0 records in 2026-03-20T11:21:34.903 INFO:teuthology.orchestra.run.vm02.stderr:1+0 records out 2026-03-20T11:21:34.903 INFO:teuthology.orchestra.run.vm02.stderr:512 bytes copied, 0.000198371 s, 2.6 MB/s 2026-03-20T11:21:34.904 DEBUG:teuthology.orchestra.run.vm02:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-03-20T11:21:34.961 DEBUG:teuthology.orchestra.run.vm02:> stat /dev/vg_nvme/lv_4 2026-03-20T11:21:35.015 INFO:teuthology.orchestra.run.vm02.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-03-20T11:21:35.015 INFO:teuthology.orchestra.run.vm02.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-20T11:21:35.015 INFO:teuthology.orchestra.run.vm02.stdout:Device: 6h/6d Inode: 767 Links: 1 2026-03-20T11:21:35.015 INFO:teuthology.orchestra.run.vm02.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-20T11:21:35.015 INFO:teuthology.orchestra.run.vm02.stdout:Context: system_u:object_r:device_t:s0 2026-03-20T11:21:35.015 INFO:teuthology.orchestra.run.vm02.stdout:Access: 2026-03-20 11:21:24.343445459 +0000 2026-03-20T11:21:35.015 INFO:teuthology.orchestra.run.vm02.stdout:Modify: 2026-03-20 11:19:56.858936019 +0000 2026-03-20T11:21:35.015 INFO:teuthology.orchestra.run.vm02.stdout:Change: 2026-03-20 11:19:56.858936019 +0000 2026-03-20T11:21:35.015 INFO:teuthology.orchestra.run.vm02.stdout: Birth: 2026-03-20 11:19:56.858936019 +0000 2026-03-20T11:21:35.016 DEBUG:teuthology.orchestra.run.vm02:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-03-20T11:21:35.079 INFO:teuthology.orchestra.run.vm02.stderr:1+0 records in 2026-03-20T11:21:35.079 INFO:teuthology.orchestra.run.vm02.stderr:1+0 records out 2026-03-20T11:21:35.079 INFO:teuthology.orchestra.run.vm02.stderr:512 bytes copied, 0.000197681 s, 2.6 MB/s 2026-03-20T11:21:35.080 DEBUG:teuthology.orchestra.run.vm02:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-03-20T11:21:35.135 INFO:tasks.ceph:osd dev map: {'c1.osd.0': '/dev/vg_nvme/lv_1', 'c1.osd.1': '/dev/vg_nvme/lv_2', 'c1.osd.2': '/dev/vg_nvme/lv_3'} 2026-03-20T11:21:35.135 DEBUG:teuthology.orchestra.run.vm05:> set -ex 2026-03-20T11:21:35.135 DEBUG:teuthology.orchestra.run.vm05:> dd if=/scratch_devs of=/dev/stdout 2026-03-20T11:21:35.151 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-03-20T11:21:35.151 DEBUG:teuthology.orchestra.run.vm05:> stat /dev/vg_nvme/lv_1 2026-03-20T11:21:35.209 INFO:teuthology.orchestra.run.vm05.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-03-20T11:21:35.209 INFO:teuthology.orchestra.run.vm05.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-20T11:21:35.209 INFO:teuthology.orchestra.run.vm05.stdout:Device: 6h/6d Inode: 645 Links: 1 2026-03-20T11:21:35.209 INFO:teuthology.orchestra.run.vm05.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-20T11:21:35.209 INFO:teuthology.orchestra.run.vm05.stdout:Context: system_u:object_r:device_t:s0 2026-03-20T11:21:35.209 INFO:teuthology.orchestra.run.vm05.stdout:Access: 2026-03-20 11:21:16.860794154 +0000 2026-03-20T11:21:35.209 INFO:teuthology.orchestra.run.vm05.stdout:Modify: 2026-03-20 11:19:47.420574813 +0000 2026-03-20T11:21:35.209 INFO:teuthology.orchestra.run.vm05.stdout:Change: 2026-03-20 11:19:47.420574813 +0000 2026-03-20T11:21:35.209 INFO:teuthology.orchestra.run.vm05.stdout: Birth: 2026-03-20 11:19:47.420574813 +0000 2026-03-20T11:21:35.209 DEBUG:teuthology.orchestra.run.vm05:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-03-20T11:21:35.273 INFO:teuthology.orchestra.run.vm05.stderr:1+0 records in 2026-03-20T11:21:35.273 INFO:teuthology.orchestra.run.vm05.stderr:1+0 records out 2026-03-20T11:21:35.273 INFO:teuthology.orchestra.run.vm05.stderr:512 bytes copied, 0.000166062 s, 3.1 MB/s 2026-03-20T11:21:35.274 DEBUG:teuthology.orchestra.run.vm05:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-03-20T11:21:35.331 DEBUG:teuthology.orchestra.run.vm05:> stat /dev/vg_nvme/lv_2 2026-03-20T11:21:35.387 INFO:teuthology.orchestra.run.vm05.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-03-20T11:21:35.387 INFO:teuthology.orchestra.run.vm05.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-20T11:21:35.387 INFO:teuthology.orchestra.run.vm05.stdout:Device: 6h/6d Inode: 674 Links: 1 2026-03-20T11:21:35.387 INFO:teuthology.orchestra.run.vm05.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-20T11:21:35.387 INFO:teuthology.orchestra.run.vm05.stdout:Context: system_u:object_r:device_t:s0 2026-03-20T11:21:35.387 INFO:teuthology.orchestra.run.vm05.stdout:Access: 2026-03-20 11:21:16.861794155 +0000 2026-03-20T11:21:35.387 INFO:teuthology.orchestra.run.vm05.stdout:Modify: 2026-03-20 11:19:47.660575113 +0000 2026-03-20T11:21:35.387 INFO:teuthology.orchestra.run.vm05.stdout:Change: 2026-03-20 11:19:47.660575113 +0000 2026-03-20T11:21:35.387 INFO:teuthology.orchestra.run.vm05.stdout: Birth: 2026-03-20 11:19:47.660575113 +0000 2026-03-20T11:21:35.387 DEBUG:teuthology.orchestra.run.vm05:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-03-20T11:21:35.453 INFO:teuthology.orchestra.run.vm05.stderr:1+0 records in 2026-03-20T11:21:35.453 INFO:teuthology.orchestra.run.vm05.stderr:1+0 records out 2026-03-20T11:21:35.453 INFO:teuthology.orchestra.run.vm05.stderr:512 bytes copied, 0.000146284 s, 3.5 MB/s 2026-03-20T11:21:35.454 DEBUG:teuthology.orchestra.run.vm05:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-03-20T11:21:35.509 DEBUG:teuthology.orchestra.run.vm05:> stat /dev/vg_nvme/lv_3 2026-03-20T11:21:35.567 INFO:teuthology.orchestra.run.vm05.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-03-20T11:21:35.567 INFO:teuthology.orchestra.run.vm05.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-20T11:21:35.567 INFO:teuthology.orchestra.run.vm05.stdout:Device: 6h/6d Inode: 727 Links: 1 2026-03-20T11:21:35.567 INFO:teuthology.orchestra.run.vm05.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-20T11:21:35.567 INFO:teuthology.orchestra.run.vm05.stdout:Context: system_u:object_r:device_t:s0 2026-03-20T11:21:35.567 INFO:teuthology.orchestra.run.vm05.stdout:Access: 2026-03-20 11:21:16.861794155 +0000 2026-03-20T11:21:35.567 INFO:teuthology.orchestra.run.vm05.stdout:Modify: 2026-03-20 11:19:47.954575480 +0000 2026-03-20T11:21:35.567 INFO:teuthology.orchestra.run.vm05.stdout:Change: 2026-03-20 11:19:47.954575480 +0000 2026-03-20T11:21:35.567 INFO:teuthology.orchestra.run.vm05.stdout: Birth: 2026-03-20 11:19:47.954575480 +0000 2026-03-20T11:21:35.567 DEBUG:teuthology.orchestra.run.vm05:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-03-20T11:21:35.632 INFO:teuthology.orchestra.run.vm05.stderr:1+0 records in 2026-03-20T11:21:35.632 INFO:teuthology.orchestra.run.vm05.stderr:1+0 records out 2026-03-20T11:21:35.632 INFO:teuthology.orchestra.run.vm05.stderr:512 bytes copied, 0.000134892 s, 3.8 MB/s 2026-03-20T11:21:35.633 DEBUG:teuthology.orchestra.run.vm05:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-03-20T11:21:35.690 DEBUG:teuthology.orchestra.run.vm05:> stat /dev/vg_nvme/lv_4 2026-03-20T11:21:35.748 INFO:teuthology.orchestra.run.vm05.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-03-20T11:21:35.748 INFO:teuthology.orchestra.run.vm05.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-20T11:21:35.748 INFO:teuthology.orchestra.run.vm05.stdout:Device: 6h/6d Inode: 777 Links: 1 2026-03-20T11:21:35.748 INFO:teuthology.orchestra.run.vm05.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-20T11:21:35.748 INFO:teuthology.orchestra.run.vm05.stdout:Context: system_u:object_r:device_t:s0 2026-03-20T11:21:35.748 INFO:teuthology.orchestra.run.vm05.stdout:Access: 2026-03-20 11:21:16.861794155 +0000 2026-03-20T11:21:35.748 INFO:teuthology.orchestra.run.vm05.stdout:Modify: 2026-03-20 11:19:48.243575841 +0000 2026-03-20T11:21:35.748 INFO:teuthology.orchestra.run.vm05.stdout:Change: 2026-03-20 11:19:48.243575841 +0000 2026-03-20T11:21:35.748 INFO:teuthology.orchestra.run.vm05.stdout: Birth: 2026-03-20 11:19:48.243575841 +0000 2026-03-20T11:21:35.748 DEBUG:teuthology.orchestra.run.vm05:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-03-20T11:21:35.812 INFO:teuthology.orchestra.run.vm05.stderr:1+0 records in 2026-03-20T11:21:35.812 INFO:teuthology.orchestra.run.vm05.stderr:1+0 records out 2026-03-20T11:21:35.812 INFO:teuthology.orchestra.run.vm05.stderr:512 bytes copied, 0.000130914 s, 3.9 MB/s 2026-03-20T11:21:35.813 DEBUG:teuthology.orchestra.run.vm05:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-03-20T11:21:35.871 INFO:tasks.ceph:osd dev map: {'c1.osd.3': '/dev/vg_nvme/lv_1', 'c1.osd.4': '/dev/vg_nvme/lv_2', 'c1.osd.5': '/dev/vg_nvme/lv_3'} 2026-03-20T11:21:35.871 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@vm02.local'): {'c1.osd.0': '/dev/vg_nvme/lv_1', 'c1.osd.1': '/dev/vg_nvme/lv_2', 'c1.osd.2': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@vm05.local'): {'c1.osd.3': '/dev/vg_nvme/lv_1', 'c1.osd.4': '/dev/vg_nvme/lv_2', 'c1.osd.5': '/dev/vg_nvme/lv_3'}} 2026-03-20T11:21:35.871 INFO:tasks.ceph:Generating config... 2026-03-20T11:21:35.872 INFO:tasks.ceph:[client] debug rgw = 20 2026-03-20T11:21:35.872 INFO:tasks.ceph:[client] debug rgw lifecycle = 20 2026-03-20T11:21:35.872 INFO:tasks.ceph:[client] debug rgw notification = 20 2026-03-20T11:21:35.872 INFO:tasks.ceph:[client] debug rgw sync = 20 2026-03-20T11:21:35.872 INFO:tasks.ceph:[client] rgw crypt require ssl = False 2026-03-20T11:21:35.872 INFO:tasks.ceph:[client] rgw crypt s3 kms backend = testing 2026-03-20T11:21:35.872 INFO:tasks.ceph:[client] rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= 2026-03-20T11:21:35.872 INFO:tasks.ceph:[client] rgw curl low speed time = 300 2026-03-20T11:21:35.872 INFO:tasks.ceph:[client] rgw data log num shards = 4 2026-03-20T11:21:35.872 INFO:tasks.ceph:[client] rgw data sync poll interval = 5 2026-03-20T11:21:35.872 INFO:tasks.ceph:[client] rgw md log max shards = 4 2026-03-20T11:21:35.872 INFO:tasks.ceph:[client] rgw meta sync poll interval = 5 2026-03-20T11:21:35.872 INFO:tasks.ceph:[client] rgw s3 auth use sts = True 2026-03-20T11:21:35.872 INFO:tasks.ceph:[client] rgw sts key = abcdefghijklmnoq 2026-03-20T11:21:35.872 INFO:tasks.ceph:[client] rgw sync data inject err probability = 0 2026-03-20T11:21:35.872 INFO:tasks.ceph:[client] rgw sync log trim interval = 0 2026-03-20T11:21:35.872 INFO:tasks.ceph:[client] rgw sync meta inject err probability = 0 2026-03-20T11:21:35.872 INFO:tasks.ceph:[client] rgw sync obj etag verify = True 2026-03-20T11:21:35.872 INFO:tasks.ceph:[client] setgroup = ceph 2026-03-20T11:21:35.872 INFO:tasks.ceph:[client] setuser = ceph 2026-03-20T11:21:35.872 INFO:tasks.ceph:[client.0] rgw data notify interval msec = 0 2026-03-20T11:21:35.872 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-03-20T11:21:35.872 INFO:tasks.ceph:[mgr] debug ms = 1 2026-03-20T11:21:35.872 INFO:tasks.ceph:[mon] debug mon = 20 2026-03-20T11:21:35.872 INFO:tasks.ceph:[mon] debug ms = 1 2026-03-20T11:21:35.872 INFO:tasks.ceph:[mon] debug paxos = 20 2026-03-20T11:21:35.872 INFO:tasks.ceph:[osd] debug ms = 1 2026-03-20T11:21:35.872 INFO:tasks.ceph:[osd] debug osd = 20 2026-03-20T11:21:35.872 INFO:tasks.ceph:[osd] osd mclock iops capacity threshold hdd = 49000 2026-03-20T11:21:35.872 INFO:tasks.ceph:[osd] rocksdb delete range threshold = 0 2026-03-20T11:21:35.872 INFO:tasks.ceph:[osd.0] osd_max_omap_entries_per_request = 10 2026-03-20T11:21:35.872 INFO:tasks.ceph:[osd.1] osd_max_omap_entries_per_request = 1000 2026-03-20T11:21:35.872 INFO:tasks.ceph:[osd.2] osd_max_omap_entries_per_request = 10000 2026-03-20T11:21:35.873 INFO:tasks.ceph:Setting up c1.mon.a... 2026-03-20T11:21:35.873 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/c1.keyring 2026-03-20T11:21:35.910 INFO:teuthology.orchestra.run.vm02.stdout:creating /etc/ceph/c1.keyring 2026-03-20T11:21:35.914 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/c1.keyring 2026-03-20T11:21:35.995 DEBUG:teuthology.orchestra.run.vm02:> sudo chmod 0644 /etc/ceph/c1.keyring 2026-03-20T11:21:36.057 DEBUG:tasks.ceph:Ceph mon addresses: [('c1.mon.a', '192.168.123.102')] 2026-03-20T11:21:36.057 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '192.168.123.102'}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'debug ms': 1, 'debug osd': 20, 'osd mclock iops capacity threshold hdd': 49000, 'rocksdb delete range threshold': 0}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rgw': 20, 'debug rgw lifecycle': 20, 'debug rgw notification': 20, 'debug rgw sync': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=', 'rgw curl low speed time': 300, 'rgw data log num shards': 4, 'rgw data sync poll interval': 5, 'rgw md log max shards': 4, 'rgw meta sync poll interval': 5, 'rgw s3 auth use sts': True, 'rgw sts key': 'abcdefghijklmnoq', 'rgw sync data inject err probability': 0, 'rgw sync log trim interval': 0, 'rgw sync meta inject err probability': 0, 'rgw sync obj etag verify': True, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'mon.a': {}, 'client.0': {'rgw data notify interval msec': 0}, 'osd.0': {'osd_max_omap_entries_per_request': 10}, 'osd.1': {'osd_max_omap_entries_per_request': 1000}, 'osd.2': {'osd_max_omap_entries_per_request': 10000}} 2026-03-20T11:21:36.058 DEBUG:teuthology.orchestra.run.vm02:> set -ex 2026-03-20T11:21:36.058 DEBUG:teuthology.orchestra.run.vm02:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-03-20T11:21:36.111 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 192.168.123.102 --print /home/ubuntu/cephtest/c1.monmap 2026-03-20T11:21:36.186 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:21:36.186 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:21:36.186 INFO:teuthology.orchestra.run.vm02.stdout:monmaptool: monmap file /home/ubuntu/cephtest/c1.monmap 2026-03-20T11:21:36.186 INFO:teuthology.orchestra.run.vm02.stdout:monmaptool: generated fsid 2b7ab434-5d47-4b50-8ba5-d98b712aaeda 2026-03-20T11:21:36.186 INFO:teuthology.orchestra.run.vm02.stdout:setting min_mon_release = tentacle 2026-03-20T11:21:36.186 INFO:teuthology.orchestra.run.vm02.stdout:epoch 0 2026-03-20T11:21:36.186 INFO:teuthology.orchestra.run.vm02.stdout:fsid 2b7ab434-5d47-4b50-8ba5-d98b712aaeda 2026-03-20T11:21:36.186 INFO:teuthology.orchestra.run.vm02.stdout:last_changed 2026-03-20T11:21:36.189072+0000 2026-03-20T11:21:36.186 INFO:teuthology.orchestra.run.vm02.stdout:created 2026-03-20T11:21:36.189072+0000 2026-03-20T11:21:36.186 INFO:teuthology.orchestra.run.vm02.stdout:min_mon_release 20 (tentacle) 2026-03-20T11:21:36.186 INFO:teuthology.orchestra.run.vm02.stdout:election_strategy: 1 2026-03-20T11:21:36.186 INFO:teuthology.orchestra.run.vm02.stdout:0: [v2:192.168.123.102:3300/0,v1:192.168.123.102:6789/0] mon.a 2026-03-20T11:21:36.186 INFO:teuthology.orchestra.run.vm02.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/c1.monmap (1 monitors) 2026-03-20T11:21:36.188 DEBUG:teuthology.orchestra.run.vm02:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-03-20T11:21:36.245 INFO:tasks.ceph:Writing /etc/ceph/c1.conf for FSID 2b7ab434-5d47-4b50-8ba5-d98b712aaeda... 2026-03-20T11:21:36.245 DEBUG:teuthology.orchestra.run.vm02:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/c1.conf && sudo chmod 0644 /etc/ceph/c1.conf > /dev/null 2026-03-20T11:21:36.286 DEBUG:teuthology.orchestra.run.vm05:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/c1.conf && sudo chmod 0644 /etc/ceph/c1.conf > /dev/null 2026-03-20T11:21:36.288 DEBUG:teuthology.orchestra.run.vm07:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/c1.conf && sudo chmod 0644 /etc/ceph/c1.conf > /dev/null 2026-03-20T11:21:36.290 DEBUG:teuthology.orchestra.run.vm08:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/c1.conf && sudo chmod 0644 /etc/ceph/c1.conf > /dev/null 2026-03-20T11:21:36.333 INFO:teuthology.orchestra.run.vm02.stdout:[global] 2026-03-20T11:21:36.333 INFO:teuthology.orchestra.run.vm02.stdout: chdir = "" 2026-03-20T11:21:36.333 INFO:teuthology.orchestra.run.vm02.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-03-20T11:21:36.333 INFO:teuthology.orchestra.run.vm02.stdout: auth supported = cephx 2026-03-20T11:21:36.333 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: filestore xattr use omap = true 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: mon clock drift allowed = 1.000 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: osd crush chooseleaf type = 0 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: auth debug = true 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: ms die on old message = true 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: ms die on bug = true 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: mon max pg per osd = 10000 # >= luminous 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: mon pg warn max object skew = 0 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: # disable pg_autoscaler by default for new pools 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: osd_pool_default_pg_autoscale_mode = off 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: osd pool default size = 2 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: mon osd allow primary affinity = true 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: mon osd allow pg remap = true 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: mon warn on legacy crush tunables = false 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: mon warn on crush straw calc version zero = false 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: mon warn on no sortbitwise = false 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: mon warn on osd down out interval zero = false 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: mon warn on too few osds = false 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: mon_warn_on_pool_no_redundancy = false 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: mon_allow_pool_size_one = true 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: osd default data pool replay window = 5 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: mon allow pool delete = true 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: mon cluster log file level = debug 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: debug asserts on shutdown = true 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: mon health detail to clog = false 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: mon host = 192.168.123.102 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: fsid = 2b7ab434-5d47-4b50-8ba5-d98b712aaeda 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout:[osd] 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: osd journal size = 100 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: osd scrub load threshold = 5.0 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: osd scrub max interval = 600 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: osd mclock profile = high_recovery_ops 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: osd mclock skip benchmark = true 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: osd recover clone overlap = true 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: osd recovery max chunk = 1048576 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: osd debug shutdown = true 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: osd debug op order = true 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: osd debug verify stray on activate = true 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: osd debug trim objects = true 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.334 INFO:teuthology.orchestra.run.vm02.stdout: osd open classes on start = true 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: osd debug pg log writeout = true 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: osd deep scrub update digest min age = 30 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: osd map max advance = 10 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: journal zero on create = true 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: filestore ondisk finisher threads = 3 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: filestore apply finisher threads = 3 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: bdev debug aio = true 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: osd debug misdirected ops = true 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: debug ms = 1 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: debug osd = 20 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: osd mclock iops capacity threshold hdd = 49000 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: rocksdb delete range threshold = 0 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout:[mgr] 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: debug ms = 1 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: debug mgr = 20 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: debug mon = 20 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: debug auth = 20 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: mon reweight min pgs per osd = 4 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: mon reweight min bytes per osd = 10 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: mgr/telemetry/nag = false 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout:[mon] 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: debug ms = 1 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: debug mon = 20 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: debug paxos = 20 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: debug auth = 20 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: mon data avail warn = 5 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: mon mgr mkfs grace = 240 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: mon reweight min pgs per osd = 4 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: mon osd reporter subtree level = osd 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: mon osd prime pg temp = true 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: mon reweight min bytes per osd = 10 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: auth mon ticket ttl = 660 # 11m 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: auth service ticket ttl = 240 # 4m 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: # don't complain about insecure global_id in the test suite 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: # 1m isn't quite enough 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: mon_down_mkfs_grace = 2m 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: mon_warn_on_filestore_osds = false 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout:[client] 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: rgw cache enabled = true 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: rgw enable ops log = true 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: rgw enable usage log = true 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: debug rgw = 20 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: debug rgw lifecycle = 20 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: debug rgw notification = 20 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: debug rgw sync = 20 2026-03-20T11:21:36.335 INFO:teuthology.orchestra.run.vm02.stdout: rgw crypt require ssl = False 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm02.stdout: rgw crypt s3 kms backend = testing 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm02.stdout: rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm02.stdout: rgw curl low speed time = 300 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm02.stdout: rgw data log num shards = 4 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm02.stdout: rgw data sync poll interval = 5 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm02.stdout: rgw md log max shards = 4 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm02.stdout: rgw meta sync poll interval = 5 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm02.stdout: rgw s3 auth use sts = True 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm02.stdout: rgw sts key = abcdefghijklmnoq 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm02.stdout: rgw sync data inject err probability = 0 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm02.stdout: rgw sync log trim interval = 0 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm02.stdout: rgw sync meta inject err probability = 0 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm02.stdout: rgw sync obj etag verify = True 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm02.stdout: setgroup = ceph 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm02.stdout: setuser = ceph 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm02.stdout:[mon.a] 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm02.stdout:[client.0] 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm02.stdout: rgw data notify interval msec = 0 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm02.stdout:[osd.0] 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm02.stdout: osd_max_omap_entries_per_request = 10 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm02.stdout:[osd.1] 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm02.stdout: osd_max_omap_entries_per_request = 1000 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm02.stdout:[osd.2] 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm02.stdout: osd_max_omap_entries_per_request = 10000 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm07.stdout:[global] 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm07.stdout: chdir = "" 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm07.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm07.stdout: auth supported = cephx 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm07.stdout: filestore xattr use omap = true 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm07.stdout: mon clock drift allowed = 1.000 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm07.stdout: osd crush chooseleaf type = 0 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm07.stdout: auth debug = true 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm07.stdout: ms die on old message = true 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm07.stdout: ms die on bug = true 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm07.stdout: mon max pg per osd = 10000 # >= luminous 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm07.stdout: mon pg warn max object skew = 0 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm07.stdout: # disable pg_autoscaler by default for new pools 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm07.stdout: osd_pool_default_pg_autoscale_mode = off 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm07.stdout: osd pool default size = 2 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm07.stdout: mon osd allow primary affinity = true 2026-03-20T11:21:36.336 INFO:teuthology.orchestra.run.vm07.stdout: mon osd allow pg remap = true 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: mon warn on legacy crush tunables = false 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: mon warn on crush straw calc version zero = false 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: mon warn on no sortbitwise = false 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: mon warn on osd down out interval zero = false 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: mon warn on too few osds = false 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: mon_warn_on_pool_no_redundancy = false 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: mon_allow_pool_size_one = true 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: osd default data pool replay window = 5 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: mon allow pool delete = true 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: mon cluster log file level = debug 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: debug asserts on shutdown = true 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: mon health detail to clog = false 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: mon host = 192.168.123.102 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: fsid = 2b7ab434-5d47-4b50-8ba5-d98b712aaeda 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout:[osd] 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: osd journal size = 100 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: osd scrub load threshold = 5.0 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: osd scrub max interval = 600 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: osd mclock profile = high_recovery_ops 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: osd mclock skip benchmark = true 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: osd recover clone overlap = true 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: osd recovery max chunk = 1048576 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: osd debug shutdown = true 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: osd debug op order = true 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: osd debug verify stray on activate = true 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: osd debug trim objects = true 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: osd open classes on start = true 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: osd debug pg log writeout = true 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: osd deep scrub update digest min age = 30 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: osd map max advance = 10 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: journal zero on create = true 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: filestore ondisk finisher threads = 3 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: filestore apply finisher threads = 3 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: bdev debug aio = true 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: osd debug misdirected ops = true 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: debug ms = 1 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: debug osd = 20 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: osd mclock iops capacity threshold hdd = 49000 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: rocksdb delete range threshold = 0 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout:[mgr] 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: debug ms = 1 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: debug mgr = 20 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: debug mon = 20 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: debug auth = 20 2026-03-20T11:21:36.337 INFO:teuthology.orchestra.run.vm07.stdout: mon reweight min pgs per osd = 4 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: mon reweight min bytes per osd = 10 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: mgr/telemetry/nag = false 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout:[mon] 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: debug ms = 1 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: debug mon = 20 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: debug paxos = 20 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: debug auth = 20 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: mon data avail warn = 5 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: mon mgr mkfs grace = 240 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: mon reweight min pgs per osd = 4 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: mon osd reporter subtree level = osd 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: mon osd prime pg temp = true 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: mon reweight min bytes per osd = 10 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: auth mon ticket ttl = 660 # 11m 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: auth service ticket ttl = 240 # 4m 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: # don't complain about insecure global_id in the test suite 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: # 1m isn't quite enough 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: mon_down_mkfs_grace = 2m 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: mon_warn_on_filestore_osds = false 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout:[client] 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: rgw cache enabled = true 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: rgw enable ops log = true 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: rgw enable usage log = true 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: debug rgw = 20 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: debug rgw lifecycle = 20 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: debug rgw notification = 20 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: debug rgw sync = 20 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: rgw crypt require ssl = False 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: rgw crypt s3 kms backend = testing 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: rgw curl low speed time = 300 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: rgw data log num shards = 4 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: rgw data sync poll interval = 5 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: rgw md log max shards = 4 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: rgw meta sync poll interval = 5 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: rgw s3 auth use sts = True 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: rgw sts key = abcdefghijklmnoq 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: rgw sync data inject err probability = 0 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: rgw sync log trim interval = 0 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: rgw sync meta inject err probability = 0 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: rgw sync obj etag verify = True 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: setgroup = ceph 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: setuser = ceph 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout:[mon.a] 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout:[client.0] 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: rgw data notify interval msec = 0 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout:[osd.0] 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: osd_max_omap_entries_per_request = 10 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout:[osd.1] 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: osd_max_omap_entries_per_request = 1000 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout:[osd.2] 2026-03-20T11:21:36.338 INFO:teuthology.orchestra.run.vm07.stdout: osd_max_omap_entries_per_request = 10000 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout:[global] 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: chdir = "" 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: auth supported = cephx 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: filestore xattr use omap = true 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: mon clock drift allowed = 1.000 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: osd crush chooseleaf type = 0 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: auth debug = true 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: ms die on old message = true 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: ms die on bug = true 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: mon max pg per osd = 10000 # >= luminous 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: mon pg warn max object skew = 0 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: # disable pg_autoscaler by default for new pools 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: osd_pool_default_pg_autoscale_mode = off 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: osd pool default size = 2 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: mon osd allow primary affinity = true 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: mon osd allow pg remap = true 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: mon warn on legacy crush tunables = false 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: mon warn on crush straw calc version zero = false 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: mon warn on no sortbitwise = false 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: mon warn on osd down out interval zero = false 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: mon warn on too few osds = false 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: mon_warn_on_pool_no_redundancy = false 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: mon_allow_pool_size_one = true 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: osd default data pool replay window = 5 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: mon allow pool delete = true 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: mon cluster log file level = debug 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: debug asserts on shutdown = true 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: mon health detail to clog = false 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: mon host = 192.168.123.102 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: fsid = 2b7ab434-5d47-4b50-8ba5-d98b712aaeda 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout:[osd] 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: osd journal size = 100 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: osd scrub load threshold = 5.0 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: osd scrub max interval = 600 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: osd mclock profile = high_recovery_ops 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: osd mclock skip benchmark = true 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: osd recover clone overlap = true 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: osd recovery max chunk = 1048576 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: osd debug shutdown = true 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: osd debug op order = true 2026-03-20T11:21:36.339 INFO:teuthology.orchestra.run.vm08.stdout: osd debug verify stray on activate = true 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: osd debug trim objects = true 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: osd open classes on start = true 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: osd debug pg log writeout = true 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: osd deep scrub update digest min age = 30 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: osd map max advance = 10 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: journal zero on create = true 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: filestore ondisk finisher threads = 3 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: filestore apply finisher threads = 3 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: bdev debug aio = true 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: osd debug misdirected ops = true 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: debug ms = 1 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: debug osd = 20 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: osd mclock iops capacity threshold hdd = 49000 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: rocksdb delete range threshold = 0 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout:[mgr] 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: debug ms = 1 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: debug mgr = 20 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: debug mon = 20 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: debug auth = 20 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: mon reweight min pgs per osd = 4 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: mon reweight min bytes per osd = 10 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: mgr/telemetry/nag = false 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout:[mon] 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: debug ms = 1 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: debug mon = 20 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: debug paxos = 20 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: debug auth = 20 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: mon data avail warn = 5 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: mon mgr mkfs grace = 240 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: mon reweight min pgs per osd = 4 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: mon osd reporter subtree level = osd 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: mon osd prime pg temp = true 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: mon reweight min bytes per osd = 10 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: auth mon ticket ttl = 660 # 11m 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: auth service ticket ttl = 240 # 4m 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: # don't complain about insecure global_id in the test suite 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: # 1m isn't quite enough 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: mon_down_mkfs_grace = 2m 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: mon_warn_on_filestore_osds = false 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout:[client] 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: rgw cache enabled = true 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: rgw enable ops log = true 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: rgw enable usage log = true 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: debug rgw = 20 2026-03-20T11:21:36.340 INFO:teuthology.orchestra.run.vm08.stdout: debug rgw lifecycle = 20 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm08.stdout: debug rgw notification = 20 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm08.stdout: debug rgw sync = 20 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm08.stdout: rgw crypt require ssl = False 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm08.stdout: rgw crypt s3 kms backend = testing 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm08.stdout: rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm08.stdout: rgw curl low speed time = 300 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm08.stdout: rgw data log num shards = 4 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm08.stdout: rgw data sync poll interval = 5 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm08.stdout: rgw md log max shards = 4 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm08.stdout: rgw meta sync poll interval = 5 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm08.stdout: rgw s3 auth use sts = True 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm08.stdout: rgw sts key = abcdefghijklmnoq 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm08.stdout: rgw sync data inject err probability = 0 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm08.stdout: rgw sync log trim interval = 0 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm08.stdout: rgw sync meta inject err probability = 0 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm08.stdout: rgw sync obj etag verify = True 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm08.stdout: setgroup = ceph 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm08.stdout: setuser = ceph 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm08.stdout:[mon.a] 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm08.stdout:[client.0] 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm08.stdout: rgw data notify interval msec = 0 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm08.stdout:[osd.0] 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm08.stdout: osd_max_omap_entries_per_request = 10 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm08.stdout:[osd.1] 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm08.stdout: osd_max_omap_entries_per_request = 1000 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm08.stdout:[osd.2] 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm08.stdout: osd_max_omap_entries_per_request = 10000 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm05.stdout:[global] 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm05.stdout: chdir = "" 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm05.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm05.stdout: auth supported = cephx 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm05.stdout: filestore xattr use omap = true 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm05.stdout: mon clock drift allowed = 1.000 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm05.stdout: osd crush chooseleaf type = 0 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm05.stdout: auth debug = true 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm05.stdout: ms die on old message = true 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm05.stdout: ms die on bug = true 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm05.stdout: mon max pg per osd = 10000 # >= luminous 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm05.stdout: mon pg warn max object skew = 0 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm05.stdout: # disable pg_autoscaler by default for new pools 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm05.stdout: osd_pool_default_pg_autoscale_mode = off 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm05.stdout: osd pool default size = 2 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm05.stdout: mon osd allow primary affinity = true 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm05.stdout: mon osd allow pg remap = true 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm05.stdout: mon warn on legacy crush tunables = false 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm05.stdout: mon warn on crush straw calc version zero = false 2026-03-20T11:21:36.341 INFO:teuthology.orchestra.run.vm05.stdout: mon warn on no sortbitwise = false 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: mon warn on osd down out interval zero = false 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: mon warn on too few osds = false 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: mon_warn_on_pool_no_redundancy = false 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: mon_allow_pool_size_one = true 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: osd default data pool replay window = 5 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: mon allow pool delete = true 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: mon cluster log file level = debug 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: debug asserts on shutdown = true 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: mon health detail to clog = false 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: mon host = 192.168.123.102 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: fsid = 2b7ab434-5d47-4b50-8ba5-d98b712aaeda 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout:[osd] 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: osd journal size = 100 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: osd scrub load threshold = 5.0 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: osd scrub max interval = 600 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: osd mclock profile = high_recovery_ops 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: osd mclock skip benchmark = true 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: osd recover clone overlap = true 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: osd recovery max chunk = 1048576 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: osd debug shutdown = true 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: osd debug op order = true 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: osd debug verify stray on activate = true 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: osd debug trim objects = true 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: osd open classes on start = true 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: osd debug pg log writeout = true 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: osd deep scrub update digest min age = 30 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: osd map max advance = 10 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: journal zero on create = true 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: filestore ondisk finisher threads = 3 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: filestore apply finisher threads = 3 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: bdev debug aio = true 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: osd debug misdirected ops = true 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: debug ms = 1 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: debug osd = 20 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: osd mclock iops capacity threshold hdd = 49000 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: rocksdb delete range threshold = 0 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout:[mgr] 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: debug ms = 1 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: debug mgr = 20 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: debug mon = 20 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: debug auth = 20 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: mon reweight min pgs per osd = 4 2026-03-20T11:21:36.342 INFO:teuthology.orchestra.run.vm05.stdout: mon reweight min bytes per osd = 10 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: mgr/telemetry/nag = false 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout:[mon] 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: debug ms = 1 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: debug mon = 20 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: debug paxos = 20 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: debug auth = 20 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: mon data avail warn = 5 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: mon mgr mkfs grace = 240 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: mon reweight min pgs per osd = 4 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: mon osd reporter subtree level = osd 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: mon osd prime pg temp = true 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: mon reweight min bytes per osd = 10 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: auth mon ticket ttl = 660 # 11m 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: auth service ticket ttl = 240 # 4m 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: # don't complain about insecure global_id in the test suite 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: # 1m isn't quite enough 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: mon_down_mkfs_grace = 2m 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: mon_warn_on_filestore_osds = false 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout:[client] 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: rgw cache enabled = true 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: rgw enable ops log = true 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: rgw enable usage log = true 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: debug rgw = 20 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: debug rgw lifecycle = 20 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: debug rgw notification = 20 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: debug rgw sync = 20 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: rgw crypt require ssl = False 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: rgw crypt s3 kms backend = testing 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: rgw curl low speed time = 300 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: rgw data log num shards = 4 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: rgw data sync poll interval = 5 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: rgw md log max shards = 4 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: rgw meta sync poll interval = 5 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: rgw s3 auth use sts = True 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: rgw sts key = abcdefghijklmnoq 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: rgw sync data inject err probability = 0 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: rgw sync log trim interval = 0 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: rgw sync meta inject err probability = 0 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: rgw sync obj etag verify = True 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: setgroup = ceph 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: setuser = ceph 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout:[mon.a] 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout:[client.0] 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: rgw data notify interval msec = 0 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout:[osd.0] 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: osd_max_omap_entries_per_request = 10 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout:[osd.1] 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: osd_max_omap_entries_per_request = 1000 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout:[osd.2] 2026-03-20T11:21:36.343 INFO:teuthology.orchestra.run.vm05.stdout: osd_max_omap_entries_per_request = 10000 2026-03-20T11:21:36.351 INFO:tasks.ceph:Creating admin key on c1.mon.a... 2026-03-20T11:21:36.351 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/c1.keyring 2026-03-20T11:21:36.435 INFO:tasks.ceph:Copying monmap to all nodes... 2026-03-20T11:21:36.435 DEBUG:teuthology.orchestra.run.vm02:> set -ex 2026-03-20T11:21:36.435 DEBUG:teuthology.orchestra.run.vm02:> dd if=/etc/ceph/c1.keyring of=/dev/stdout 2026-03-20T11:21:36.451 DEBUG:teuthology.orchestra.run.vm02:> set -ex 2026-03-20T11:21:36.451 DEBUG:teuthology.orchestra.run.vm02:> dd if=/home/ubuntu/cephtest/c1.monmap of=/dev/stdout 2026-03-20T11:21:36.507 INFO:tasks.ceph:Sending monmap to node ubuntu@vm02.local 2026-03-20T11:21:36.507 DEBUG:teuthology.orchestra.run.vm02:> set -ex 2026-03-20T11:21:36.507 DEBUG:teuthology.orchestra.run.vm02:> sudo dd of=/etc/ceph/c1.keyring 2026-03-20T11:21:36.507 DEBUG:teuthology.orchestra.run.vm02:> sudo chmod 0644 /etc/ceph/c1.keyring 2026-03-20T11:21:36.582 DEBUG:teuthology.orchestra.run.vm02:> set -ex 2026-03-20T11:21:36.582 DEBUG:teuthology.orchestra.run.vm02:> dd of=/home/ubuntu/cephtest/c1.monmap 2026-03-20T11:21:36.637 INFO:tasks.ceph:Sending monmap to node ubuntu@vm05.local 2026-03-20T11:21:36.637 DEBUG:teuthology.orchestra.run.vm05:> set -ex 2026-03-20T11:21:36.637 DEBUG:teuthology.orchestra.run.vm05:> sudo dd of=/etc/ceph/c1.keyring 2026-03-20T11:21:36.637 DEBUG:teuthology.orchestra.run.vm05:> sudo chmod 0644 /etc/ceph/c1.keyring 2026-03-20T11:21:36.675 DEBUG:teuthology.orchestra.run.vm05:> set -ex 2026-03-20T11:21:36.675 DEBUG:teuthology.orchestra.run.vm05:> dd of=/home/ubuntu/cephtest/c1.monmap 2026-03-20T11:21:36.733 INFO:tasks.ceph:Sending monmap to node ubuntu@vm07.local 2026-03-20T11:21:36.733 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-20T11:21:36.733 DEBUG:teuthology.orchestra.run.vm07:> sudo dd of=/etc/ceph/c1.keyring 2026-03-20T11:21:36.733 DEBUG:teuthology.orchestra.run.vm07:> sudo chmod 0644 /etc/ceph/c1.keyring 2026-03-20T11:21:36.770 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-20T11:21:36.770 DEBUG:teuthology.orchestra.run.vm07:> dd of=/home/ubuntu/cephtest/c1.monmap 2026-03-20T11:21:36.827 INFO:tasks.ceph:Sending monmap to node ubuntu@vm08.local 2026-03-20T11:21:36.827 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-20T11:21:36.827 DEBUG:teuthology.orchestra.run.vm08:> sudo dd of=/etc/ceph/c1.keyring 2026-03-20T11:21:36.827 DEBUG:teuthology.orchestra.run.vm08:> sudo chmod 0644 /etc/ceph/c1.keyring 2026-03-20T11:21:36.860 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-20T11:21:36.860 DEBUG:teuthology.orchestra.run.vm08:> dd of=/home/ubuntu/cephtest/c1.monmap 2026-03-20T11:21:36.915 INFO:tasks.ceph:Setting up mon nodes... 2026-03-20T11:21:36.915 INFO:tasks.ceph:Setting up mgr nodes... 2026-03-20T11:21:36.915 DEBUG:teuthology.orchestra.run.vm05:> sudo mkdir -p /var/lib/ceph/mgr/c1-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/c1-x/keyring 2026-03-20T11:21:36.969 INFO:teuthology.orchestra.run.vm05.stdout:creating /var/lib/ceph/mgr/c1-x/keyring 2026-03-20T11:21:36.972 INFO:tasks.ceph:Setting up mds nodes... 2026-03-20T11:21:36.972 INFO:tasks.ceph_client:Setting up client nodes... 2026-03-20T11:21:36.972 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/c1.client.0.keyring && sudo chmod 0644 /etc/ceph/c1.client.0.keyring 2026-03-20T11:21:37.011 INFO:teuthology.orchestra.run.vm02.stdout:creating /etc/ceph/c1.client.0.keyring 2026-03-20T11:21:37.021 DEBUG:teuthology.orchestra.run.vm05:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.1 /etc/ceph/c1.client.1.keyring && sudo chmod 0644 /etc/ceph/c1.client.1.keyring 2026-03-20T11:21:37.061 INFO:teuthology.orchestra.run.vm05.stdout:creating /etc/ceph/c1.client.1.keyring 2026-03-20T11:21:37.074 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-03-20T11:21:37.074 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@vm02.local'): {'c1.osd.0': '/dev/vg_nvme/lv_1', 'c1.osd.1': '/dev/vg_nvme/lv_2', 'c1.osd.2': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@vm05.local'): {'c1.osd.3': '/dev/vg_nvme/lv_1', 'c1.osd.4': '/dev/vg_nvme/lv_2', 'c1.osd.5': '/dev/vg_nvme/lv_3'}} 2026-03-20T11:21:37.074 DEBUG:teuthology.orchestra.run.vm02:> sudo mkdir -p /var/lib/ceph/osd/c1-0 2026-03-20T11:21:37.099 INFO:tasks.ceph:roles_to_devs: {'c1.osd.0': '/dev/vg_nvme/lv_1', 'c1.osd.1': '/dev/vg_nvme/lv_2', 'c1.osd.2': '/dev/vg_nvme/lv_3'} 2026-03-20T11:21:37.099 INFO:tasks.ceph:role: c1.osd.0 2026-03-20T11:21:37.099 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@vm02.local 2026-03-20T11:21:37.099 DEBUG:teuthology.orchestra.run.vm02:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-03-20T11:21:37.167 INFO:teuthology.orchestra.run.vm02.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=1310464 blks 2026-03-20T11:21:37.167 INFO:teuthology.orchestra.run.vm02.stdout: = sectsz=512 attr=2, projid32bit=1 2026-03-20T11:21:37.167 INFO:teuthology.orchestra.run.vm02.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-03-20T11:21:37.167 INFO:teuthology.orchestra.run.vm02.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-03-20T11:21:37.167 INFO:teuthology.orchestra.run.vm02.stdout:data = bsize=4096 blocks=5241856, imaxpct=25 2026-03-20T11:21:37.167 INFO:teuthology.orchestra.run.vm02.stdout: = sunit=0 swidth=0 blks 2026-03-20T11:21:37.167 INFO:teuthology.orchestra.run.vm02.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-03-20T11:21:37.167 INFO:teuthology.orchestra.run.vm02.stdout:log =internal log bsize=4096 blocks=16384, version=2 2026-03-20T11:21:37.167 INFO:teuthology.orchestra.run.vm02.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2026-03-20T11:21:37.167 INFO:teuthology.orchestra.run.vm02.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-03-20T11:21:37.177 INFO:teuthology.orchestra.run.vm02.stdout:Discarding blocks...Done. 2026-03-20T11:21:37.182 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@vm02.local -o noatime 2026-03-20T11:21:37.182 DEBUG:teuthology.orchestra.run.vm02:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/c1-0 2026-03-20T11:21:37.256 DEBUG:teuthology.orchestra.run.vm02:> sudo /sbin/restorecon /var/lib/ceph/osd/c1-0 2026-03-20T11:21:37.324 DEBUG:teuthology.orchestra.run.vm02:> sudo mkdir -p /var/lib/ceph/osd/c1-1 2026-03-20T11:21:37.389 INFO:tasks.ceph:roles_to_devs: {'c1.osd.0': '/dev/vg_nvme/lv_1', 'c1.osd.1': '/dev/vg_nvme/lv_2', 'c1.osd.2': '/dev/vg_nvme/lv_3'} 2026-03-20T11:21:37.389 INFO:tasks.ceph:role: c1.osd.1 2026-03-20T11:21:37.389 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@vm02.local 2026-03-20T11:21:37.390 DEBUG:teuthology.orchestra.run.vm02:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-03-20T11:21:37.457 INFO:teuthology.orchestra.run.vm02.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=1310464 blks 2026-03-20T11:21:37.457 INFO:teuthology.orchestra.run.vm02.stdout: = sectsz=512 attr=2, projid32bit=1 2026-03-20T11:21:37.457 INFO:teuthology.orchestra.run.vm02.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-03-20T11:21:37.457 INFO:teuthology.orchestra.run.vm02.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-03-20T11:21:37.457 INFO:teuthology.orchestra.run.vm02.stdout:data = bsize=4096 blocks=5241856, imaxpct=25 2026-03-20T11:21:37.457 INFO:teuthology.orchestra.run.vm02.stdout: = sunit=0 swidth=0 blks 2026-03-20T11:21:37.457 INFO:teuthology.orchestra.run.vm02.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-03-20T11:21:37.457 INFO:teuthology.orchestra.run.vm02.stdout:log =internal log bsize=4096 blocks=16384, version=2 2026-03-20T11:21:37.458 INFO:teuthology.orchestra.run.vm02.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2026-03-20T11:21:37.458 INFO:teuthology.orchestra.run.vm02.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-03-20T11:21:37.463 INFO:teuthology.orchestra.run.vm02.stdout:Discarding blocks...Done. 2026-03-20T11:21:37.465 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@vm02.local -o noatime 2026-03-20T11:21:37.465 DEBUG:teuthology.orchestra.run.vm02:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/c1-1 2026-03-20T11:21:37.533 DEBUG:teuthology.orchestra.run.vm02:> sudo /sbin/restorecon /var/lib/ceph/osd/c1-1 2026-03-20T11:21:37.600 DEBUG:teuthology.orchestra.run.vm02:> sudo mkdir -p /var/lib/ceph/osd/c1-2 2026-03-20T11:21:37.666 INFO:tasks.ceph:roles_to_devs: {'c1.osd.0': '/dev/vg_nvme/lv_1', 'c1.osd.1': '/dev/vg_nvme/lv_2', 'c1.osd.2': '/dev/vg_nvme/lv_3'} 2026-03-20T11:21:37.666 INFO:tasks.ceph:role: c1.osd.2 2026-03-20T11:21:37.666 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@vm02.local 2026-03-20T11:21:37.666 DEBUG:teuthology.orchestra.run.vm02:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-03-20T11:21:37.732 INFO:teuthology.orchestra.run.vm02.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=1310464 blks 2026-03-20T11:21:37.732 INFO:teuthology.orchestra.run.vm02.stdout: = sectsz=512 attr=2, projid32bit=1 2026-03-20T11:21:37.732 INFO:teuthology.orchestra.run.vm02.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-03-20T11:21:37.732 INFO:teuthology.orchestra.run.vm02.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-03-20T11:21:37.732 INFO:teuthology.orchestra.run.vm02.stdout:data = bsize=4096 blocks=5241856, imaxpct=25 2026-03-20T11:21:37.732 INFO:teuthology.orchestra.run.vm02.stdout: = sunit=0 swidth=0 blks 2026-03-20T11:21:37.732 INFO:teuthology.orchestra.run.vm02.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-03-20T11:21:37.732 INFO:teuthology.orchestra.run.vm02.stdout:log =internal log bsize=4096 blocks=16384, version=2 2026-03-20T11:21:37.732 INFO:teuthology.orchestra.run.vm02.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2026-03-20T11:21:37.732 INFO:teuthology.orchestra.run.vm02.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-03-20T11:21:37.737 INFO:teuthology.orchestra.run.vm02.stdout:Discarding blocks...Done. 2026-03-20T11:21:37.738 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@vm02.local -o noatime 2026-03-20T11:21:37.738 DEBUG:teuthology.orchestra.run.vm02:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/c1-2 2026-03-20T11:21:37.821 DEBUG:teuthology.orchestra.run.vm02:> sudo /sbin/restorecon /var/lib/ceph/osd/c1-2 2026-03-20T11:21:37.889 DEBUG:teuthology.orchestra.run.vm02:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster c1 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/c1.monmap 2026-03-20T11:21:37.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:21:37.976+0000 7fcd13f89900 -1 auth: error reading file: /var/lib/ceph/osd/c1-0/keyring: can't open /var/lib/ceph/osd/c1-0/keyring: (2) No such file or directory 2026-03-20T11:21:37.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:21:37.976+0000 7fcd13f89900 -1 created new key in keyring /var/lib/ceph/osd/c1-0/keyring 2026-03-20T11:21:37.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:21:37.976+0000 7fcd13f89900 -1 bdev(0x55bdb74b9800 /var/lib/ceph/osd/c1-0/block) open stat got: (1) Operation not permitted 2026-03-20T11:21:37.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:21:37.976+0000 7fcd13f89900 -1 bluestore(/var/lib/ceph/osd/c1-0) _read_fsid unparsable uuid 2026-03-20T11:21:38.471 DEBUG:teuthology.orchestra.run.vm02:> sudo chown -R ceph:ceph /var/lib/ceph/osd/c1-0 2026-03-20T11:21:38.495 DEBUG:teuthology.orchestra.run.vm02:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster c1 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/c1.monmap 2026-03-20T11:21:38.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:21:38.578+0000 7fc8571dc900 -1 auth: error reading file: /var/lib/ceph/osd/c1-1/keyring: can't open /var/lib/ceph/osd/c1-1/keyring: (2) No such file or directory 2026-03-20T11:21:38.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:21:38.578+0000 7fc8571dc900 -1 created new key in keyring /var/lib/ceph/osd/c1-1/keyring 2026-03-20T11:21:38.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:21:38.578+0000 7fc8571dc900 -1 bdev(0x564102b1d800 /var/lib/ceph/osd/c1-1/block) open stat got: (1) Operation not permitted 2026-03-20T11:21:38.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:21:38.578+0000 7fc8571dc900 -1 bluestore(/var/lib/ceph/osd/c1-1) _read_fsid unparsable uuid 2026-03-20T11:21:39.017 DEBUG:teuthology.orchestra.run.vm02:> sudo chown -R ceph:ceph /var/lib/ceph/osd/c1-1 2026-03-20T11:21:39.081 DEBUG:teuthology.orchestra.run.vm02:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster c1 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/c1.monmap 2026-03-20T11:21:39.160 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:21:39.161+0000 7fa63b8f7900 -1 auth: error reading file: /var/lib/ceph/osd/c1-2/keyring: can't open /var/lib/ceph/osd/c1-2/keyring: (2) No such file or directory 2026-03-20T11:21:39.160 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:21:39.161+0000 7fa63b8f7900 -1 created new key in keyring /var/lib/ceph/osd/c1-2/keyring 2026-03-20T11:21:39.160 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:21:39.161+0000 7fa63b8f7900 -1 bdev(0x556e8b199800 /var/lib/ceph/osd/c1-2/block) open stat got: (1) Operation not permitted 2026-03-20T11:21:39.160 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:21:39.161+0000 7fa63b8f7900 -1 bluestore(/var/lib/ceph/osd/c1-2) _read_fsid unparsable uuid 2026-03-20T11:21:39.613 DEBUG:teuthology.orchestra.run.vm02:> sudo chown -R ceph:ceph /var/lib/ceph/osd/c1-2 2026-03-20T11:21:39.677 DEBUG:teuthology.orchestra.run.vm05:> sudo mkdir -p /var/lib/ceph/osd/c1-3 2026-03-20T11:21:39.706 INFO:tasks.ceph:roles_to_devs: {'c1.osd.3': '/dev/vg_nvme/lv_1', 'c1.osd.4': '/dev/vg_nvme/lv_2', 'c1.osd.5': '/dev/vg_nvme/lv_3'} 2026-03-20T11:21:39.706 INFO:tasks.ceph:role: c1.osd.3 2026-03-20T11:21:39.706 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@vm05.local 2026-03-20T11:21:39.706 DEBUG:teuthology.orchestra.run.vm05:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-03-20T11:21:39.774 INFO:teuthology.orchestra.run.vm05.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=1310464 blks 2026-03-20T11:21:39.774 INFO:teuthology.orchestra.run.vm05.stdout: = sectsz=512 attr=2, projid32bit=1 2026-03-20T11:21:39.774 INFO:teuthology.orchestra.run.vm05.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-03-20T11:21:39.774 INFO:teuthology.orchestra.run.vm05.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-03-20T11:21:39.774 INFO:teuthology.orchestra.run.vm05.stdout:data = bsize=4096 blocks=5241856, imaxpct=25 2026-03-20T11:21:39.774 INFO:teuthology.orchestra.run.vm05.stdout: = sunit=0 swidth=0 blks 2026-03-20T11:21:39.774 INFO:teuthology.orchestra.run.vm05.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-03-20T11:21:39.774 INFO:teuthology.orchestra.run.vm05.stdout:log =internal log bsize=4096 blocks=16384, version=2 2026-03-20T11:21:39.774 INFO:teuthology.orchestra.run.vm05.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2026-03-20T11:21:39.774 INFO:teuthology.orchestra.run.vm05.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-03-20T11:21:39.779 INFO:teuthology.orchestra.run.vm05.stdout:Discarding blocks...Done. 2026-03-20T11:21:39.782 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@vm05.local -o noatime 2026-03-20T11:21:39.782 DEBUG:teuthology.orchestra.run.vm05:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/c1-3 2026-03-20T11:21:39.853 DEBUG:teuthology.orchestra.run.vm05:> sudo /sbin/restorecon /var/lib/ceph/osd/c1-3 2026-03-20T11:21:39.925 DEBUG:teuthology.orchestra.run.vm05:> sudo mkdir -p /var/lib/ceph/osd/c1-4 2026-03-20T11:21:39.991 INFO:tasks.ceph:roles_to_devs: {'c1.osd.3': '/dev/vg_nvme/lv_1', 'c1.osd.4': '/dev/vg_nvme/lv_2', 'c1.osd.5': '/dev/vg_nvme/lv_3'} 2026-03-20T11:21:39.991 INFO:tasks.ceph:role: c1.osd.4 2026-03-20T11:21:39.991 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@vm05.local 2026-03-20T11:21:39.991 DEBUG:teuthology.orchestra.run.vm05:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-03-20T11:21:40.057 INFO:teuthology.orchestra.run.vm05.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=1310464 blks 2026-03-20T11:21:40.057 INFO:teuthology.orchestra.run.vm05.stdout: = sectsz=512 attr=2, projid32bit=1 2026-03-20T11:21:40.057 INFO:teuthology.orchestra.run.vm05.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-03-20T11:21:40.057 INFO:teuthology.orchestra.run.vm05.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-03-20T11:21:40.057 INFO:teuthology.orchestra.run.vm05.stdout:data = bsize=4096 blocks=5241856, imaxpct=25 2026-03-20T11:21:40.057 INFO:teuthology.orchestra.run.vm05.stdout: = sunit=0 swidth=0 blks 2026-03-20T11:21:40.057 INFO:teuthology.orchestra.run.vm05.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-03-20T11:21:40.057 INFO:teuthology.orchestra.run.vm05.stdout:log =internal log bsize=4096 blocks=16384, version=2 2026-03-20T11:21:40.057 INFO:teuthology.orchestra.run.vm05.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2026-03-20T11:21:40.057 INFO:teuthology.orchestra.run.vm05.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-03-20T11:21:40.062 INFO:teuthology.orchestra.run.vm05.stdout:Discarding blocks...Done. 2026-03-20T11:21:40.064 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@vm05.local -o noatime 2026-03-20T11:21:40.064 DEBUG:teuthology.orchestra.run.vm05:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/c1-4 2026-03-20T11:21:40.134 DEBUG:teuthology.orchestra.run.vm05:> sudo /sbin/restorecon /var/lib/ceph/osd/c1-4 2026-03-20T11:21:40.203 DEBUG:teuthology.orchestra.run.vm05:> sudo mkdir -p /var/lib/ceph/osd/c1-5 2026-03-20T11:21:40.268 INFO:tasks.ceph:roles_to_devs: {'c1.osd.3': '/dev/vg_nvme/lv_1', 'c1.osd.4': '/dev/vg_nvme/lv_2', 'c1.osd.5': '/dev/vg_nvme/lv_3'} 2026-03-20T11:21:40.268 INFO:tasks.ceph:role: c1.osd.5 2026-03-20T11:21:40.268 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@vm05.local 2026-03-20T11:21:40.268 DEBUG:teuthology.orchestra.run.vm05:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-03-20T11:21:40.332 INFO:teuthology.orchestra.run.vm05.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=1310464 blks 2026-03-20T11:21:40.332 INFO:teuthology.orchestra.run.vm05.stdout: = sectsz=512 attr=2, projid32bit=1 2026-03-20T11:21:40.332 INFO:teuthology.orchestra.run.vm05.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-03-20T11:21:40.332 INFO:teuthology.orchestra.run.vm05.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-03-20T11:21:40.332 INFO:teuthology.orchestra.run.vm05.stdout:data = bsize=4096 blocks=5241856, imaxpct=25 2026-03-20T11:21:40.333 INFO:teuthology.orchestra.run.vm05.stdout: = sunit=0 swidth=0 blks 2026-03-20T11:21:40.333 INFO:teuthology.orchestra.run.vm05.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-03-20T11:21:40.333 INFO:teuthology.orchestra.run.vm05.stdout:log =internal log bsize=4096 blocks=16384, version=2 2026-03-20T11:21:40.333 INFO:teuthology.orchestra.run.vm05.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2026-03-20T11:21:40.333 INFO:teuthology.orchestra.run.vm05.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-03-20T11:21:40.337 INFO:teuthology.orchestra.run.vm05.stdout:Discarding blocks...Done. 2026-03-20T11:21:40.340 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@vm05.local -o noatime 2026-03-20T11:21:40.340 DEBUG:teuthology.orchestra.run.vm05:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/c1-5 2026-03-20T11:21:40.410 DEBUG:teuthology.orchestra.run.vm05:> sudo /sbin/restorecon /var/lib/ceph/osd/c1-5 2026-03-20T11:21:40.482 DEBUG:teuthology.orchestra.run.vm05:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster c1 --mkfs --mkkey -i 3 --monmap /home/ubuntu/cephtest/c1.monmap 2026-03-20T11:21:40.566 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:21:40.568+0000 7f801de72900 -1 auth: error reading file: /var/lib/ceph/osd/c1-3/keyring: can't open /var/lib/ceph/osd/c1-3/keyring: (2) No such file or directory 2026-03-20T11:21:40.567 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:21:40.568+0000 7f801de72900 -1 created new key in keyring /var/lib/ceph/osd/c1-3/keyring 2026-03-20T11:21:40.567 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:21:40.568+0000 7f801de72900 -1 bdev(0x55a6e1b11800 /var/lib/ceph/osd/c1-3/block) open stat got: (1) Operation not permitted 2026-03-20T11:21:40.567 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:21:40.568+0000 7f801de72900 -1 bluestore(/var/lib/ceph/osd/c1-3) _read_fsid unparsable uuid 2026-03-20T11:21:41.005 DEBUG:teuthology.orchestra.run.vm05:> sudo chown -R ceph:ceph /var/lib/ceph/osd/c1-3 2026-03-20T11:21:41.036 DEBUG:teuthology.orchestra.run.vm05:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster c1 --mkfs --mkkey -i 4 --monmap /home/ubuntu/cephtest/c1.monmap 2026-03-20T11:21:41.116 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:21:41.118+0000 7f047de95900 -1 auth: error reading file: /var/lib/ceph/osd/c1-4/keyring: can't open /var/lib/ceph/osd/c1-4/keyring: (2) No such file or directory 2026-03-20T11:21:41.117 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:21:41.118+0000 7f047de95900 -1 created new key in keyring /var/lib/ceph/osd/c1-4/keyring 2026-03-20T11:21:41.117 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:21:41.118+0000 7f047de95900 -1 bdev(0x56521da0f800 /var/lib/ceph/osd/c1-4/block) open stat got: (1) Operation not permitted 2026-03-20T11:21:41.117 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:21:41.118+0000 7f047de95900 -1 bluestore(/var/lib/ceph/osd/c1-4) _read_fsid unparsable uuid 2026-03-20T11:21:41.584 DEBUG:teuthology.orchestra.run.vm05:> sudo chown -R ceph:ceph /var/lib/ceph/osd/c1-4 2026-03-20T11:21:41.652 DEBUG:teuthology.orchestra.run.vm05:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster c1 --mkfs --mkkey -i 5 --monmap /home/ubuntu/cephtest/c1.monmap 2026-03-20T11:21:41.734 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:21:41.736+0000 7f72a39c1900 -1 auth: error reading file: /var/lib/ceph/osd/c1-5/keyring: can't open /var/lib/ceph/osd/c1-5/keyring: (2) No such file or directory 2026-03-20T11:21:41.735 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:21:41.736+0000 7f72a39c1900 -1 created new key in keyring /var/lib/ceph/osd/c1-5/keyring 2026-03-20T11:21:41.735 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:21:41.736+0000 7f72a39c1900 -1 bdev(0x55f9ac815800 /var/lib/ceph/osd/c1-5/block) open stat got: (1) Operation not permitted 2026-03-20T11:21:41.735 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:21:41.736+0000 7f72a39c1900 -1 bluestore(/var/lib/ceph/osd/c1-5) _read_fsid unparsable uuid 2026-03-20T11:21:42.468 DEBUG:teuthology.orchestra.run.vm05:> sudo chown -R ceph:ceph /var/lib/ceph/osd/c1-5 2026-03-20T11:21:42.534 INFO:tasks.ceph:Reading keys from all nodes... 2026-03-20T11:21:42.535 DEBUG:teuthology.orchestra.run.vm02:> set -ex 2026-03-20T11:21:42.535 DEBUG:teuthology.orchestra.run.vm02:> sudo dd if=/var/lib/ceph/osd/c1-0/keyring of=/dev/stdout 2026-03-20T11:21:42.558 DEBUG:teuthology.orchestra.run.vm02:> set -ex 2026-03-20T11:21:42.570 DEBUG:teuthology.orchestra.run.vm02:> sudo dd if=/var/lib/ceph/osd/c1-1/keyring of=/dev/stdout 2026-03-20T11:21:42.622 DEBUG:teuthology.orchestra.run.vm02:> set -ex 2026-03-20T11:21:42.622 DEBUG:teuthology.orchestra.run.vm02:> sudo dd if=/var/lib/ceph/osd/c1-2/keyring of=/dev/stdout 2026-03-20T11:21:42.686 DEBUG:teuthology.orchestra.run.vm05:> set -ex 2026-03-20T11:21:42.686 DEBUG:teuthology.orchestra.run.vm05:> sudo dd if=/var/lib/ceph/mgr/c1-x/keyring of=/dev/stdout 2026-03-20T11:21:42.716 DEBUG:teuthology.orchestra.run.vm05:> set -ex 2026-03-20T11:21:42.716 DEBUG:teuthology.orchestra.run.vm05:> sudo dd if=/var/lib/ceph/osd/c1-3/keyring of=/dev/stdout 2026-03-20T11:21:42.784 DEBUG:teuthology.orchestra.run.vm05:> set -ex 2026-03-20T11:21:42.784 DEBUG:teuthology.orchestra.run.vm05:> sudo dd if=/var/lib/ceph/osd/c1-4/keyring of=/dev/stdout 2026-03-20T11:21:42.853 DEBUG:teuthology.orchestra.run.vm05:> set -ex 2026-03-20T11:21:42.853 DEBUG:teuthology.orchestra.run.vm05:> sudo dd if=/var/lib/ceph/osd/c1-5/keyring of=/dev/stdout 2026-03-20T11:21:42.921 DEBUG:teuthology.orchestra.run.vm02:> set -ex 2026-03-20T11:21:42.921 DEBUG:teuthology.orchestra.run.vm02:> dd if=/etc/ceph/c1.client.0.keyring of=/dev/stdout 2026-03-20T11:21:42.935 DEBUG:teuthology.orchestra.run.vm05:> set -ex 2026-03-20T11:21:42.935 DEBUG:teuthology.orchestra.run.vm05:> dd if=/etc/ceph/c1.client.1.keyring of=/dev/stdout 2026-03-20T11:21:42.981 INFO:tasks.ceph:Adding keys to all mons... 2026-03-20T11:21:42.981 DEBUG:teuthology.orchestra.run.vm02:> sudo tee -a /etc/ceph/c1.keyring 2026-03-20T11:21:43.004 INFO:teuthology.orchestra.run.vm02.stdout:[osd.0] 2026-03-20T11:21:43.005 INFO:teuthology.orchestra.run.vm02.stdout: key = AQDBLb1pLB9JOhAAI1mHSEYXxSwY/1iD9nlh+Q== 2026-03-20T11:21:43.005 INFO:teuthology.orchestra.run.vm02.stdout:[osd.1] 2026-03-20T11:21:43.005 INFO:teuthology.orchestra.run.vm02.stdout: key = AQDCLb1pjTCNIhAAZv+3k5/DggPkeYD3tpDAFw== 2026-03-20T11:21:43.005 INFO:teuthology.orchestra.run.vm02.stdout:[osd.2] 2026-03-20T11:21:43.005 INFO:teuthology.orchestra.run.vm02.stdout: key = AQDDLb1pvXuuCRAA2mR+F8rXRJnVe1oFkd90rQ== 2026-03-20T11:21:43.005 INFO:teuthology.orchestra.run.vm02.stdout:[mgr.x] 2026-03-20T11:21:43.005 INFO:teuthology.orchestra.run.vm02.stdout: key = AQDALb1p8z/qORAAWHSjOoqT/A1H9u+LhP9RRw== 2026-03-20T11:21:43.005 INFO:teuthology.orchestra.run.vm02.stdout:[osd.3] 2026-03-20T11:21:43.005 INFO:teuthology.orchestra.run.vm02.stdout: key = AQDELb1pWDbtIRAAhglgVglKMUbepWdWB8Rp5g== 2026-03-20T11:21:43.005 INFO:teuthology.orchestra.run.vm02.stdout:[osd.4] 2026-03-20T11:21:43.005 INFO:teuthology.orchestra.run.vm02.stdout: key = AQDFLb1pqm4YBxAAXUqG22hHzTXtYDnxxCrRwQ== 2026-03-20T11:21:43.005 INFO:teuthology.orchestra.run.vm02.stdout:[osd.5] 2026-03-20T11:21:43.005 INFO:teuthology.orchestra.run.vm02.stdout: key = AQDFLb1pGxjtKxAAJoAm5xUwq4iXpb0wr6t/GQ== 2026-03-20T11:21:43.005 INFO:teuthology.orchestra.run.vm02.stdout:[client.0] 2026-03-20T11:21:43.005 INFO:teuthology.orchestra.run.vm02.stdout: key = AQDBLb1pBVDRABAA99c9N+iVxJEAKq20R6+liQ== 2026-03-20T11:21:43.005 INFO:teuthology.orchestra.run.vm02.stdout:[client.1] 2026-03-20T11:21:43.005 INFO:teuthology.orchestra.run.vm02.stdout: key = AQDBLb1pSwDPAxAAYTaAoT+aU1YVmk+KyrrwFA== 2026-03-20T11:21:43.005 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c1.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-03-20T11:21:43.091 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c1.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-03-20T11:21:43.136 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c1.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-03-20T11:21:43.177 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c1.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-03-20T11:21:43.266 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c1.keyring --name=osd.3 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-03-20T11:21:43.312 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c1.keyring --name=osd.4 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-03-20T11:21:43.399 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c1.keyring --name=osd.5 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-03-20T11:21:43.442 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c1.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-20T11:21:43.526 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c1.keyring --name=client.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-20T11:21:43.574 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-03-20T11:21:43.575 DEBUG:teuthology.orchestra.run.vm02:> sudo mkdir -p /var/lib/ceph/mon/c1-a 2026-03-20T11:21:43.599 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster c1 --mkfs -i a --monmap /home/ubuntu/cephtest/c1.monmap --keyring /etc/ceph/c1.keyring 2026-03-20T11:21:43.816 DEBUG:teuthology.orchestra.run.vm02:> sudo chown -R ceph:ceph /var/lib/ceph/mon/c1-a 2026-03-20T11:21:43.846 DEBUG:teuthology.orchestra.run.vm02:> rm -- /home/ubuntu/cephtest/c1.monmap 2026-03-20T11:21:43.902 INFO:tasks.ceph:Starting mon daemons in cluster c1... 2026-03-20T11:21:43.903 INFO:tasks.ceph.c1.mon.a:Restarting daemon 2026-03-20T11:21:43.903 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster c1 -i a 2026-03-20T11:21:43.945 INFO:tasks.ceph.c1.mon.a:Started 2026-03-20T11:21:43.945 INFO:tasks.ceph:Starting mgr daemons in cluster c1... 2026-03-20T11:21:43.945 INFO:tasks.ceph.c1.mgr.x:Restarting daemon 2026-03-20T11:21:43.945 DEBUG:teuthology.orchestra.run.vm05:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster c1 -i x 2026-03-20T11:21:43.947 INFO:tasks.ceph.c1.mgr.x:Started 2026-03-20T11:21:43.947 DEBUG:tasks.ceph:set 0 configs 2026-03-20T11:21:43.947 DEBUG:teuthology.orchestra.run.vm02:> sudo ceph --cluster c1 config dump 2026-03-20T11:21:44.255 INFO:teuthology.orchestra.run.vm02.stdout:WHO MASK LEVEL OPTION VALUE RO 2026-03-20T11:21:44.264 INFO:tasks.ceph:Setting crush tunables to default 2026-03-20T11:21:44.264 DEBUG:teuthology.orchestra.run.vm02:> sudo ceph --cluster c1 osd crush tunables default 2026-03-20T11:21:44.376 INFO:teuthology.orchestra.run.vm02.stderr:adjusted tunables profile to default 2026-03-20T11:21:44.387 INFO:tasks.ceph:check_enable_crimson: False 2026-03-20T11:21:44.387 INFO:tasks.ceph:Starting osd daemons in cluster c1... 2026-03-20T11:21:44.387 DEBUG:teuthology.orchestra.run.vm02:> set -ex 2026-03-20T11:21:44.387 DEBUG:teuthology.orchestra.run.vm02:> sudo dd if=/var/lib/ceph/osd/c1-0/fsid of=/dev/stdout 2026-03-20T11:21:44.413 DEBUG:teuthology.orchestra.run.vm02:> set -ex 2026-03-20T11:21:44.413 DEBUG:teuthology.orchestra.run.vm02:> sudo dd if=/var/lib/ceph/osd/c1-1/fsid of=/dev/stdout 2026-03-20T11:21:44.477 DEBUG:teuthology.orchestra.run.vm02:> set -ex 2026-03-20T11:21:44.477 DEBUG:teuthology.orchestra.run.vm02:> sudo dd if=/var/lib/ceph/osd/c1-2/fsid of=/dev/stdout 2026-03-20T11:21:44.541 DEBUG:teuthology.orchestra.run.vm05:> set -ex 2026-03-20T11:21:44.541 DEBUG:teuthology.orchestra.run.vm05:> sudo dd if=/var/lib/ceph/osd/c1-3/fsid of=/dev/stdout 2026-03-20T11:21:44.572 DEBUG:teuthology.orchestra.run.vm05:> set -ex 2026-03-20T11:21:44.572 DEBUG:teuthology.orchestra.run.vm05:> sudo dd if=/var/lib/ceph/osd/c1-4/fsid of=/dev/stdout 2026-03-20T11:21:44.640 DEBUG:teuthology.orchestra.run.vm05:> set -ex 2026-03-20T11:21:44.640 DEBUG:teuthology.orchestra.run.vm05:> sudo dd if=/var/lib/ceph/osd/c1-5/fsid of=/dev/stdout 2026-03-20T11:21:44.707 DEBUG:teuthology.orchestra.run.vm05:> sudo ceph --cluster c1 osd new a02501c9-9a7c-457f-a76d-5e24da486b1e 0 2026-03-20T11:21:44.863 INFO:teuthology.orchestra.run.vm05.stdout:0 2026-03-20T11:21:44.872 DEBUG:teuthology.orchestra.run.vm05:> sudo ceph --cluster c1 osd new 5bf6ce37-6b8c-48c2-bd17-1dd5eec74076 1 2026-03-20T11:21:44.986 INFO:teuthology.orchestra.run.vm05.stdout:1 2026-03-20T11:21:44.996 DEBUG:teuthology.orchestra.run.vm05:> sudo ceph --cluster c1 osd new 7a13d281-5a55-40a2-8b1a-c050cfe10021 2 2026-03-20T11:21:45.021 INFO:tasks.ceph.c1.mgr.x.vm05.stderr:/usr/lib64/python3.9/site-packages/scipy/__init__.py:73: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-03-20T11:21:45.021 INFO:tasks.ceph.c1.mgr.x.vm05.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-03-20T11:21:45.021 INFO:tasks.ceph.c1.mgr.x.vm05.stderr: from numpy import show_config as show_numpy_config 2026-03-20T11:21:45.110 INFO:teuthology.orchestra.run.vm05.stdout:2 2026-03-20T11:21:45.120 DEBUG:teuthology.orchestra.run.vm05:> sudo ceph --cluster c1 osd new e38d88c7-cb9d-4933-8620-282da1599052 3 2026-03-20T11:21:45.277 INFO:teuthology.orchestra.run.vm05.stdout:3 2026-03-20T11:21:45.287 DEBUG:teuthology.orchestra.run.vm05:> sudo ceph --cluster c1 osd new 1df74db1-22c6-4c38-9a55-e52de1d25b40 4 2026-03-20T11:21:45.403 INFO:teuthology.orchestra.run.vm05.stdout:4 2026-03-20T11:21:45.413 DEBUG:teuthology.orchestra.run.vm05:> sudo ceph --cluster c1 osd new a448fa2d-2798-46c0-a2ec-02ee5301f820 5 2026-03-20T11:21:45.525 INFO:teuthology.orchestra.run.vm05.stdout:5 2026-03-20T11:21:45.534 INFO:tasks.ceph.c1.osd.0:Restarting daemon 2026-03-20T11:21:45.534 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster c1 -i 0 2026-03-20T11:21:45.535 INFO:tasks.ceph.c1.osd.0:Started 2026-03-20T11:21:45.535 INFO:tasks.ceph.c1.osd.1:Restarting daemon 2026-03-20T11:21:45.535 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster c1 -i 1 2026-03-20T11:21:45.537 INFO:tasks.ceph.c1.osd.1:Started 2026-03-20T11:21:45.537 INFO:tasks.ceph.c1.osd.2:Restarting daemon 2026-03-20T11:21:45.537 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster c1 -i 2 2026-03-20T11:21:45.539 INFO:tasks.ceph.c1.osd.2:Started 2026-03-20T11:21:45.539 INFO:tasks.ceph.c1.osd.3:Restarting daemon 2026-03-20T11:21:45.539 DEBUG:teuthology.orchestra.run.vm05:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster c1 -i 3 2026-03-20T11:21:45.540 INFO:tasks.ceph.c1.osd.3:Started 2026-03-20T11:21:45.540 INFO:tasks.ceph.c1.osd.4:Restarting daemon 2026-03-20T11:21:45.540 DEBUG:teuthology.orchestra.run.vm05:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster c1 -i 4 2026-03-20T11:21:45.541 INFO:tasks.ceph.c1.osd.4:Started 2026-03-20T11:21:45.541 INFO:tasks.ceph.c1.osd.5:Restarting daemon 2026-03-20T11:21:45.541 DEBUG:teuthology.orchestra.run.vm05:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster c1 -i 5 2026-03-20T11:21:45.547 INFO:tasks.ceph.c1.osd.5:Started 2026-03-20T11:21:45.548 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd dump --format=json 2026-03-20T11:21:45.675 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:45.675 INFO:teuthology.orchestra.run.vm02.stdout:{"epoch":8,"fsid":"2b7ab434-5d47-4b50-8ba5-d98b712aaeda","created":"2026-03-20T11:21:44.070502+0000","modified":"2026-03-20T11:21:45.525871+0000","last_up_change":"0.000000","last_in_change":"2026-03-20T11:21:45.525871+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":6,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"a02501c9-9a7c-457f-a76d-5e24da486b1e","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"5bf6ce37-6b8c-48c2-bd17-1dd5eec74076","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"7a13d281-5a55-40a2-8b1a-c050cfe10021","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"e38d88c7-cb9d-4933-8620-282da1599052","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":4,"uuid":"1df74db1-22c6-4c38-9a55-e52de1d25b40","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":5,"uuid":"a448fa2d-2798-46c0-a2ec-02ee5301f820","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-20T11:21:45.685 INFO:tasks.ceph.ceph_manager.c1:[] 2026-03-20T11:21:45.685 INFO:tasks.ceph:Waiting for OSDs to come up 2026-03-20T11:21:45.712 INFO:tasks.ceph.c1.osd.1.vm02.stderr:2026-03-20T11:21:45.713+0000 7faaf6da0900 -1 Falling back to public interface 2026-03-20T11:21:45.717 INFO:tasks.ceph.c1.osd.0.vm02.stderr:2026-03-20T11:21:45.718+0000 7ff5c1813900 -1 Falling back to public interface 2026-03-20T11:21:45.722 INFO:tasks.ceph.c1.osd.2.vm02.stderr:2026-03-20T11:21:45.724+0000 7fb94ca5b900 -1 Falling back to public interface 2026-03-20T11:21:45.749 INFO:tasks.ceph.c1.osd.5.vm05.stderr:2026-03-20T11:21:45.750+0000 7f2ccd410900 -1 Falling back to public interface 2026-03-20T11:21:45.765 INFO:tasks.ceph.c1.osd.3.vm05.stderr:2026-03-20T11:21:45.766+0000 7f70fa810900 -1 Falling back to public interface 2026-03-20T11:21:45.765 INFO:tasks.ceph.c1.osd.4.vm05.stderr:2026-03-20T11:21:45.767+0000 7fae57741900 -1 Falling back to public interface 2026-03-20T11:21:45.766 INFO:tasks.ceph.c1.osd.4.vm05.stderr:2026-03-20T11:21:45.768+0000 7fae566e9640 -1 PosixStack listen unable to listen on v2:0.0.0.0:6816/0: (98) Address already in use 2026-03-20T11:21:45.824 INFO:tasks.ceph.c1.osd.1.vm02.stderr:2026-03-20T11:21:45.825+0000 7faaf6da0900 -1 osd.1 0 log_to_monitors true 2026-03-20T11:21:45.845 INFO:tasks.ceph.c1.osd.0.vm02.stderr:2026-03-20T11:21:45.846+0000 7ff5c1813900 -1 osd.0 0 log_to_monitors true 2026-03-20T11:21:45.846 INFO:tasks.ceph.c1.osd.2.vm02.stderr:2026-03-20T11:21:45.847+0000 7fb94ca5b900 -1 osd.2 0 log_to_monitors true 2026-03-20T11:21:45.906 INFO:tasks.ceph.c1.osd.5.vm05.stderr:2026-03-20T11:21:45.907+0000 7f2ccd410900 -1 osd.5 0 log_to_monitors true 2026-03-20T11:21:45.929 INFO:tasks.ceph.c1.osd.3.vm05.stderr:2026-03-20T11:21:45.930+0000 7f70fa810900 -1 osd.3 0 log_to_monitors true 2026-03-20T11:21:45.938 INFO:tasks.ceph.c1.osd.4.vm05.stderr:2026-03-20T11:21:45.939+0000 7fae57741900 -1 osd.4 0 log_to_monitors true 2026-03-20T11:21:46.289 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster c1 osd dump --format=json 2026-03-20T11:21:46.393 INFO:teuthology.misc.health.vm02.stdout: 2026-03-20T11:21:46.393 INFO:teuthology.misc.health.vm02.stdout:{"epoch":8,"fsid":"2b7ab434-5d47-4b50-8ba5-d98b712aaeda","created":"2026-03-20T11:21:44.070502+0000","modified":"2026-03-20T11:21:45.525871+0000","last_up_change":"0.000000","last_in_change":"2026-03-20T11:21:45.525871+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":6,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"a02501c9-9a7c-457f-a76d-5e24da486b1e","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"5bf6ce37-6b8c-48c2-bd17-1dd5eec74076","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"7a13d281-5a55-40a2-8b1a-c050cfe10021","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"e38d88c7-cb9d-4933-8620-282da1599052","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":4,"uuid":"1df74db1-22c6-4c38-9a55-e52de1d25b40","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":5,"uuid":"a448fa2d-2798-46c0-a2ec-02ee5301f820","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-20T11:21:46.401 DEBUG:teuthology.misc:0 of 6 OSDs are up 2026-03-20T11:21:47.531 INFO:tasks.ceph.c1.osd.5.vm05.stderr:2026-03-20T11:21:47.532+0000 7f2cc8b9c640 -1 osd.5 0 waiting for initial osdmap 2026-03-20T11:21:47.531 INFO:tasks.ceph.c1.osd.0.vm02.stderr:2026-03-20T11:21:47.532+0000 7ff5bce9c640 -1 osd.0 0 waiting for initial osdmap 2026-03-20T11:21:47.531 INFO:tasks.ceph.c1.osd.4.vm05.stderr:2026-03-20T11:21:47.533+0000 7fae536d2640 -1 osd.4 0 waiting for initial osdmap 2026-03-20T11:21:47.532 INFO:tasks.ceph.c1.osd.1.vm02.stderr:2026-03-20T11:21:47.533+0000 7faaf2d2f640 -1 osd.1 0 waiting for initial osdmap 2026-03-20T11:21:47.532 INFO:tasks.ceph.c1.osd.2.vm02.stderr:2026-03-20T11:21:47.533+0000 7fb9489ea640 -1 osd.2 0 waiting for initial osdmap 2026-03-20T11:21:47.532 INFO:tasks.ceph.c1.osd.3.vm05.stderr:2026-03-20T11:21:47.533+0000 7f70f6051640 -1 osd.3 0 waiting for initial osdmap 2026-03-20T11:21:47.542 INFO:tasks.ceph.c1.osd.0.vm02.stderr:2026-03-20T11:21:47.543+0000 7ff5b7ca1640 -1 osd.0 10 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-03-20T11:21:47.544 INFO:tasks.ceph.c1.osd.4.vm05.stderr:2026-03-20T11:21:47.546+0000 7fae4e4d7640 -1 osd.4 10 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-03-20T11:21:47.544 INFO:tasks.ceph.c1.osd.3.vm05.stderr:2026-03-20T11:21:47.546+0000 7f70f0e56640 -1 osd.3 10 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-03-20T11:21:47.546 INFO:tasks.ceph.c1.osd.5.vm05.stderr:2026-03-20T11:21:47.547+0000 7f2cc39a1640 -1 osd.5 10 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-03-20T11:21:47.546 INFO:tasks.ceph.c1.osd.2.vm02.stderr:2026-03-20T11:21:47.546+0000 7fb9437ef640 -1 osd.2 10 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-03-20T11:21:47.546 INFO:tasks.ceph.c1.osd.1.vm02.stderr:2026-03-20T11:21:47.546+0000 7faaedb34640 -1 osd.1 10 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-03-20T11:21:47.548 INFO:tasks.ceph.c1.mgr.x.vm05.stderr:2026-03-20T11:21:47.549+0000 7faef8526640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-03-20T11:21:53.007 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster c1 osd dump --format=json 2026-03-20T11:21:53.196 INFO:teuthology.misc.health.vm02.stdout: 2026-03-20T11:21:53.196 INFO:teuthology.misc.health.vm02.stdout:{"epoch":15,"fsid":"2b7ab434-5d47-4b50-8ba5-d98b712aaeda","created":"2026-03-20T11:21:44.070502+0000","modified":"2026-03-20T11:21:52.616035+0000","last_up_change":"2026-03-20T11:21:48.544405+0000","last_in_change":"2026-03-20T11:21:45.525871+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":6,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-03-20T11:21:50.557473+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":6.059999942779541,"score_stable":6.059999942779541,"optimal_score":0.33000001311302185,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"a02501c9-9a7c-457f-a76d-5e24da486b1e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6808","nonce":729406321},{"type":"v1","addr":"192.168.123.102:6809","nonce":729406321}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6810","nonce":729406321},{"type":"v1","addr":"192.168.123.102:6811","nonce":729406321}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6814","nonce":729406321},{"type":"v1","addr":"192.168.123.102:6815","nonce":729406321}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6812","nonce":729406321},{"type":"v1","addr":"192.168.123.102:6813","nonce":729406321}]},"public_addr":"192.168.123.102:6809/729406321","cluster_addr":"192.168.123.102:6811/729406321","heartbeat_back_addr":"192.168.123.102:6815/729406321","heartbeat_front_addr":"192.168.123.102:6813/729406321","state":["exists","up"]},{"osd":1,"uuid":"5bf6ce37-6b8c-48c2-bd17-1dd5eec74076","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6800","nonce":3585770291},{"type":"v1","addr":"192.168.123.102:6801","nonce":3585770291}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6802","nonce":3585770291},{"type":"v1","addr":"192.168.123.102:6803","nonce":3585770291}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6806","nonce":3585770291},{"type":"v1","addr":"192.168.123.102:6807","nonce":3585770291}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6804","nonce":3585770291},{"type":"v1","addr":"192.168.123.102:6805","nonce":3585770291}]},"public_addr":"192.168.123.102:6801/3585770291","cluster_addr":"192.168.123.102:6803/3585770291","heartbeat_back_addr":"192.168.123.102:6807/3585770291","heartbeat_front_addr":"192.168.123.102:6805/3585770291","state":["exists","up"]},{"osd":2,"uuid":"7a13d281-5a55-40a2-8b1a-c050cfe10021","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6816","nonce":1902334371},{"type":"v1","addr":"192.168.123.102:6817","nonce":1902334371}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6818","nonce":1902334371},{"type":"v1","addr":"192.168.123.102:6819","nonce":1902334371}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6822","nonce":1902334371},{"type":"v1","addr":"192.168.123.102:6823","nonce":1902334371}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6820","nonce":1902334371},{"type":"v1","addr":"192.168.123.102:6821","nonce":1902334371}]},"public_addr":"192.168.123.102:6817/1902334371","cluster_addr":"192.168.123.102:6819/1902334371","heartbeat_back_addr":"192.168.123.102:6823/1902334371","heartbeat_front_addr":"192.168.123.102:6821/1902334371","state":["exists","up"]},{"osd":3,"uuid":"e38d88c7-cb9d-4933-8620-282da1599052","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6808","nonce":4120232946},{"type":"v1","addr":"192.168.123.105:6809","nonce":4120232946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6810","nonce":4120232946},{"type":"v1","addr":"192.168.123.105:6811","nonce":4120232946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6816","nonce":4120232946},{"type":"v1","addr":"192.168.123.105:6818","nonce":4120232946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6813","nonce":4120232946},{"type":"v1","addr":"192.168.123.105:6815","nonce":4120232946}]},"public_addr":"192.168.123.105:6809/4120232946","cluster_addr":"192.168.123.105:6811/4120232946","heartbeat_back_addr":"192.168.123.105:6818/4120232946","heartbeat_front_addr":"192.168.123.105:6815/4120232946","state":["exists","up"]},{"osd":4,"uuid":"1df74db1-22c6-4c38-9a55-e52de1d25b40","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6812","nonce":2020435041},{"type":"v1","addr":"192.168.123.105:6814","nonce":2020435041}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6817","nonce":2020435041},{"type":"v1","addr":"192.168.123.105:6819","nonce":2020435041}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6822","nonce":2020435041},{"type":"v1","addr":"192.168.123.105:6823","nonce":2020435041}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6820","nonce":2020435041},{"type":"v1","addr":"192.168.123.105:6821","nonce":2020435041}]},"public_addr":"192.168.123.105:6814/2020435041","cluster_addr":"192.168.123.105:6819/2020435041","heartbeat_back_addr":"192.168.123.105:6823/2020435041","heartbeat_front_addr":"192.168.123.105:6821/2020435041","state":["exists","up"]},{"osd":5,"uuid":"a448fa2d-2798-46c0-a2ec-02ee5301f820","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6800","nonce":735244627},{"type":"v1","addr":"192.168.123.105:6801","nonce":735244627}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6802","nonce":735244627},{"type":"v1","addr":"192.168.123.105:6803","nonce":735244627}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6806","nonce":735244627},{"type":"v1","addr":"192.168.123.105:6807","nonce":735244627}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6804","nonce":735244627},{"type":"v1","addr":"192.168.123.105:6805","nonce":735244627}]},"public_addr":"192.168.123.105:6801/735244627","cluster_addr":"192.168.123.105:6803/735244627","heartbeat_back_addr":"192.168.123.105:6807/735244627","heartbeat_front_addr":"192.168.123.105:6805/735244627","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:21:46.833501+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:21:46.861810+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:21:46.829544+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-20T11:21:53.209 DEBUG:teuthology.misc:6 of 6 OSDs are up 2026-03-20T11:21:53.209 INFO:tasks.ceph:Creating RBD pool 2026-03-20T11:21:53.209 DEBUG:teuthology.orchestra.run.vm02:> sudo ceph --cluster c1 osd pool create rbd 8 2026-03-20T11:21:53.747 INFO:teuthology.orchestra.run.vm02.stderr:pool 'rbd' created 2026-03-20T11:21:53.762 DEBUG:teuthology.orchestra.run.vm02:> rbd --cluster c1 pool init rbd 2026-03-20T11:21:53.820 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:21:53.821 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:21:56.772 INFO:tasks.ceph:Starting mds daemons in cluster c1... 2026-03-20T11:21:56.772 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 config log 1 --format=json 2026-03-20T11:21:56.772 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-03-20T11:21:56.960 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:56.970 INFO:teuthology.orchestra.run.vm02.stdout:[{"version":1,"timestamp":"0.000000","name":"","changes":[]}] 2026-03-20T11:21:56.970 INFO:tasks.ceph_manager:config epoch is 1 2026-03-20T11:21:56.970 INFO:tasks.ceph:Waiting until c1 daemons up and pgs clean... 2026-03-20T11:21:56.970 INFO:tasks.ceph.ceph_manager.c1:waiting for mgr available 2026-03-20T11:21:56.970 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 mgr dump --format=json 2026-03-20T11:21:57.195 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:57.206 INFO:teuthology.orchestra.run.vm02.stdout:{"epoch":5,"flags":0,"active_gid":4098,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6824","nonce":1718843890},{"type":"v1","addr":"192.168.123.105:6825","nonce":1718843890}]},"active_addr":"192.168.123.105:6825/1718843890","active_change":"2026-03-20T11:21:46.540773+0000","active_mgr_features":4544132024016699391,"available":true,"standbys":[],"modules":["iostat","nfs"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"certificate_automated_rotation_enabled":{"name":"certificate_automated_rotation_enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"This flag controls whether cephadm automatically rotates certificates upon expiration.","long_desc":"","tags":[],"see_also":[]},"certificate_check_debug_mode":{"name":"certificate_check_debug_mode","type":"bool","level":"dev","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"FOR TESTING ONLY: This flag forces the certificate check instead of waiting for certificate_check_period.","long_desc":"","tags":[],"see_also":[]},"certificate_check_period":{"name":"certificate_check_period","type":"int","level":"advanced","flags":0,"default_value":"1","min":"0","max":"30","enum_allowed":[],"desc":"Specifies how often (in days) the certificate should be checked for validity.","long_desc":"","tags":[],"see_also":[]},"certificate_duration_days":{"name":"certificate_duration_days","type":"int","level":"advanced","flags":0,"default_value":"1095","min":"90","max":"3650","enum_allowed":[],"desc":"Specifies the duration of self certificates generated and signed by cephadm root CA","long_desc":"","tags":[],"see_also":[]},"certificate_renewal_threshold_days":{"name":"certificate_renewal_threshold_days","type":"int","level":"advanced","flags":0,"default_value":"30","min":"10","max":"90","enum_allowed":[],"desc":"Specifies the lead time in days to initiate certificate renewal before expiration.","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.28.1","min":"","max":"","enum_allowed":[],"desc":"Alertmanager container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"Elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:12.3.1","min":"","max":"","enum_allowed":[],"desc":"Grafana container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"Haproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:sclorg-nginx-126","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.9.1","min":"","max":"","enum_allowed":[],"desc":"Node exporter container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.5","min":"","max":"","enum_allowed":[],"desc":"Nvmeof container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"Oauth2 proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v3.6.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba_metrics":{"name":"container_image_samba_metrics","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-metrics:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba metrics container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"Snmp gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"stray_daemon_check_interval":{"name":"stray_daemon_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"how frequently cephadm should check for the presence of stray daemons","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROM_ALERT_CREDENTIAL_CACHE_TTL":{"name":"PROM_ALERT_CREDENTIAL_CACHE_TTL","type":"int","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_HOSTNAME_PER_DAEMON":{"name":"RGW_HOSTNAME_PER_DAEMON","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crypto_caller":{"name":"crypto_caller","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sso_oauth2":{"name":"sso_oauth2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"prometheus_tls_secret_name":{"name":"prometheus_tls_secret_name","type":"str","level":"advanced","flags":0,"default_value":"rook-ceph-prometheus-server-tls","min":"","max":"","enum_allowed":[],"desc":"name of tls secret in k8s for prometheus","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_cloning":{"name":"pause_cloning","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_purging":{"name":"pause_purging","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous subvolume purge threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"tentacle":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"192.168.123.105:0","nonce":1207581178}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"192.168.123.105:0","nonce":2012284090}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"192.168.123.105:0","nonce":1178884701}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"192.168.123.105:0","nonce":3632617944}]}]} 2026-03-20T11:21:57.206 INFO:tasks.ceph.ceph_manager.c1:mgr available! 2026-03-20T11:21:57.206 INFO:tasks.ceph.ceph_manager.c1:waiting for all up 2026-03-20T11:21:57.206 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd dump --format=json 2026-03-20T11:21:57.393 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:57.393 INFO:teuthology.orchestra.run.vm02.stdout:{"epoch":19,"fsid":"2b7ab434-5d47-4b50-8ba5-d98b712aaeda","created":"2026-03-20T11:21:44.070502+0000","modified":"2026-03-20T11:21:56.762471+0000","last_up_change":"2026-03-20T11:21:48.544405+0000","last_in_change":"2026-03-20T11:21:45.525871+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":6,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-03-20T11:21:50.557473+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":6.059999942779541,"score_stable":6.059999942779541,"optimal_score":0.33000001311302185,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-03-20T11:21:53.414491+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":19,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"a02501c9-9a7c-457f-a76d-5e24da486b1e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6808","nonce":729406321},{"type":"v1","addr":"192.168.123.102:6809","nonce":729406321}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6810","nonce":729406321},{"type":"v1","addr":"192.168.123.102:6811","nonce":729406321}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6814","nonce":729406321},{"type":"v1","addr":"192.168.123.102:6815","nonce":729406321}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6812","nonce":729406321},{"type":"v1","addr":"192.168.123.102:6813","nonce":729406321}]},"public_addr":"192.168.123.102:6809/729406321","cluster_addr":"192.168.123.102:6811/729406321","heartbeat_back_addr":"192.168.123.102:6815/729406321","heartbeat_front_addr":"192.168.123.102:6813/729406321","state":["exists","up"]},{"osd":1,"uuid":"5bf6ce37-6b8c-48c2-bd17-1dd5eec74076","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6800","nonce":3585770291},{"type":"v1","addr":"192.168.123.102:6801","nonce":3585770291}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6802","nonce":3585770291},{"type":"v1","addr":"192.168.123.102:6803","nonce":3585770291}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6806","nonce":3585770291},{"type":"v1","addr":"192.168.123.102:6807","nonce":3585770291}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6804","nonce":3585770291},{"type":"v1","addr":"192.168.123.102:6805","nonce":3585770291}]},"public_addr":"192.168.123.102:6801/3585770291","cluster_addr":"192.168.123.102:6803/3585770291","heartbeat_back_addr":"192.168.123.102:6807/3585770291","heartbeat_front_addr":"192.168.123.102:6805/3585770291","state":["exists","up"]},{"osd":2,"uuid":"7a13d281-5a55-40a2-8b1a-c050cfe10021","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6816","nonce":1902334371},{"type":"v1","addr":"192.168.123.102:6817","nonce":1902334371}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6818","nonce":1902334371},{"type":"v1","addr":"192.168.123.102:6819","nonce":1902334371}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6822","nonce":1902334371},{"type":"v1","addr":"192.168.123.102:6823","nonce":1902334371}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6820","nonce":1902334371},{"type":"v1","addr":"192.168.123.102:6821","nonce":1902334371}]},"public_addr":"192.168.123.102:6817/1902334371","cluster_addr":"192.168.123.102:6819/1902334371","heartbeat_back_addr":"192.168.123.102:6823/1902334371","heartbeat_front_addr":"192.168.123.102:6821/1902334371","state":["exists","up"]},{"osd":3,"uuid":"e38d88c7-cb9d-4933-8620-282da1599052","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6808","nonce":4120232946},{"type":"v1","addr":"192.168.123.105:6809","nonce":4120232946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6810","nonce":4120232946},{"type":"v1","addr":"192.168.123.105:6811","nonce":4120232946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6816","nonce":4120232946},{"type":"v1","addr":"192.168.123.105:6818","nonce":4120232946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6813","nonce":4120232946},{"type":"v1","addr":"192.168.123.105:6815","nonce":4120232946}]},"public_addr":"192.168.123.105:6809/4120232946","cluster_addr":"192.168.123.105:6811/4120232946","heartbeat_back_addr":"192.168.123.105:6818/4120232946","heartbeat_front_addr":"192.168.123.105:6815/4120232946","state":["exists","up"]},{"osd":4,"uuid":"1df74db1-22c6-4c38-9a55-e52de1d25b40","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6812","nonce":2020435041},{"type":"v1","addr":"192.168.123.105:6814","nonce":2020435041}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6817","nonce":2020435041},{"type":"v1","addr":"192.168.123.105:6819","nonce":2020435041}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6822","nonce":2020435041},{"type":"v1","addr":"192.168.123.105:6823","nonce":2020435041}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6820","nonce":2020435041},{"type":"v1","addr":"192.168.123.105:6821","nonce":2020435041}]},"public_addr":"192.168.123.105:6814/2020435041","cluster_addr":"192.168.123.105:6819/2020435041","heartbeat_back_addr":"192.168.123.105:6823/2020435041","heartbeat_front_addr":"192.168.123.105:6821/2020435041","state":["exists","up"]},{"osd":5,"uuid":"a448fa2d-2798-46c0-a2ec-02ee5301f820","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6800","nonce":735244627},{"type":"v1","addr":"192.168.123.105:6801","nonce":735244627}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6802","nonce":735244627},{"type":"v1","addr":"192.168.123.105:6803","nonce":735244627}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6806","nonce":735244627},{"type":"v1","addr":"192.168.123.105:6807","nonce":735244627}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6804","nonce":735244627},{"type":"v1","addr":"192.168.123.105:6805","nonce":735244627}]},"public_addr":"192.168.123.105:6801/735244627","cluster_addr":"192.168.123.105:6803/735244627","heartbeat_back_addr":"192.168.123.105:6807/735244627","heartbeat_front_addr":"192.168.123.105:6805/735244627","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:21:46.833501+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:21:46.861810+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:21:46.829544+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:21:46.930054+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:21:46.921019+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:21:46.934043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-20T11:21:57.403 INFO:tasks.ceph.ceph_manager.c1:all up! 2026-03-20T11:21:57.403 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd dump --format=json 2026-03-20T11:21:57.597 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:57.598 INFO:teuthology.orchestra.run.vm02.stdout:{"epoch":19,"fsid":"2b7ab434-5d47-4b50-8ba5-d98b712aaeda","created":"2026-03-20T11:21:44.070502+0000","modified":"2026-03-20T11:21:56.762471+0000","last_up_change":"2026-03-20T11:21:48.544405+0000","last_in_change":"2026-03-20T11:21:45.525871+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":6,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-03-20T11:21:50.557473+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":6.059999942779541,"score_stable":6.059999942779541,"optimal_score":0.33000001311302185,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-03-20T11:21:53.414491+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":19,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"a02501c9-9a7c-457f-a76d-5e24da486b1e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6808","nonce":729406321},{"type":"v1","addr":"192.168.123.102:6809","nonce":729406321}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6810","nonce":729406321},{"type":"v1","addr":"192.168.123.102:6811","nonce":729406321}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6814","nonce":729406321},{"type":"v1","addr":"192.168.123.102:6815","nonce":729406321}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6812","nonce":729406321},{"type":"v1","addr":"192.168.123.102:6813","nonce":729406321}]},"public_addr":"192.168.123.102:6809/729406321","cluster_addr":"192.168.123.102:6811/729406321","heartbeat_back_addr":"192.168.123.102:6815/729406321","heartbeat_front_addr":"192.168.123.102:6813/729406321","state":["exists","up"]},{"osd":1,"uuid":"5bf6ce37-6b8c-48c2-bd17-1dd5eec74076","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6800","nonce":3585770291},{"type":"v1","addr":"192.168.123.102:6801","nonce":3585770291}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6802","nonce":3585770291},{"type":"v1","addr":"192.168.123.102:6803","nonce":3585770291}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6806","nonce":3585770291},{"type":"v1","addr":"192.168.123.102:6807","nonce":3585770291}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6804","nonce":3585770291},{"type":"v1","addr":"192.168.123.102:6805","nonce":3585770291}]},"public_addr":"192.168.123.102:6801/3585770291","cluster_addr":"192.168.123.102:6803/3585770291","heartbeat_back_addr":"192.168.123.102:6807/3585770291","heartbeat_front_addr":"192.168.123.102:6805/3585770291","state":["exists","up"]},{"osd":2,"uuid":"7a13d281-5a55-40a2-8b1a-c050cfe10021","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6816","nonce":1902334371},{"type":"v1","addr":"192.168.123.102:6817","nonce":1902334371}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6818","nonce":1902334371},{"type":"v1","addr":"192.168.123.102:6819","nonce":1902334371}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6822","nonce":1902334371},{"type":"v1","addr":"192.168.123.102:6823","nonce":1902334371}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.102:6820","nonce":1902334371},{"type":"v1","addr":"192.168.123.102:6821","nonce":1902334371}]},"public_addr":"192.168.123.102:6817/1902334371","cluster_addr":"192.168.123.102:6819/1902334371","heartbeat_back_addr":"192.168.123.102:6823/1902334371","heartbeat_front_addr":"192.168.123.102:6821/1902334371","state":["exists","up"]},{"osd":3,"uuid":"e38d88c7-cb9d-4933-8620-282da1599052","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6808","nonce":4120232946},{"type":"v1","addr":"192.168.123.105:6809","nonce":4120232946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6810","nonce":4120232946},{"type":"v1","addr":"192.168.123.105:6811","nonce":4120232946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6816","nonce":4120232946},{"type":"v1","addr":"192.168.123.105:6818","nonce":4120232946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6813","nonce":4120232946},{"type":"v1","addr":"192.168.123.105:6815","nonce":4120232946}]},"public_addr":"192.168.123.105:6809/4120232946","cluster_addr":"192.168.123.105:6811/4120232946","heartbeat_back_addr":"192.168.123.105:6818/4120232946","heartbeat_front_addr":"192.168.123.105:6815/4120232946","state":["exists","up"]},{"osd":4,"uuid":"1df74db1-22c6-4c38-9a55-e52de1d25b40","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6812","nonce":2020435041},{"type":"v1","addr":"192.168.123.105:6814","nonce":2020435041}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6817","nonce":2020435041},{"type":"v1","addr":"192.168.123.105:6819","nonce":2020435041}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6822","nonce":2020435041},{"type":"v1","addr":"192.168.123.105:6823","nonce":2020435041}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6820","nonce":2020435041},{"type":"v1","addr":"192.168.123.105:6821","nonce":2020435041}]},"public_addr":"192.168.123.105:6814/2020435041","cluster_addr":"192.168.123.105:6819/2020435041","heartbeat_back_addr":"192.168.123.105:6823/2020435041","heartbeat_front_addr":"192.168.123.105:6821/2020435041","state":["exists","up"]},{"osd":5,"uuid":"a448fa2d-2798-46c0-a2ec-02ee5301f820","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6800","nonce":735244627},{"type":"v1","addr":"192.168.123.105:6801","nonce":735244627}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6802","nonce":735244627},{"type":"v1","addr":"192.168.123.105:6803","nonce":735244627}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6806","nonce":735244627},{"type":"v1","addr":"192.168.123.105:6807","nonce":735244627}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.105:6804","nonce":735244627},{"type":"v1","addr":"192.168.123.105:6805","nonce":735244627}]},"public_addr":"192.168.123.105:6801/735244627","cluster_addr":"192.168.123.105:6803/735244627","heartbeat_back_addr":"192.168.123.105:6807/735244627","heartbeat_front_addr":"192.168.123.105:6805/735244627","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:21:46.833501+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:21:46.861810+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:21:46.829544+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:21:46.930054+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:21:46.921019+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:21:46.934043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-20T11:21:57.609 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 tell osd.0 flush_pg_stats 2026-03-20T11:21:57.609 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 tell osd.1 flush_pg_stats 2026-03-20T11:21:57.609 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 tell osd.2 flush_pg_stats 2026-03-20T11:21:57.609 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 tell osd.3 flush_pg_stats 2026-03-20T11:21:57.609 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 tell osd.4 flush_pg_stats 2026-03-20T11:21:57.609 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 tell osd.5 flush_pg_stats 2026-03-20T11:21:57.768 INFO:teuthology.orchestra.run.vm02.stdout:47244640259 2026-03-20T11:21:57.769 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd last-stat-seq osd.5 2026-03-20T11:21:57.770 INFO:teuthology.orchestra.run.vm02.stdout:47244640259 2026-03-20T11:21:57.770 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd last-stat-seq osd.2 2026-03-20T11:21:57.770 INFO:teuthology.orchestra.run.vm02.stdout:47244640259 2026-03-20T11:21:57.770 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd last-stat-seq osd.4 2026-03-20T11:21:57.774 INFO:teuthology.orchestra.run.vm02.stdout:47244640259 2026-03-20T11:21:57.774 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd last-stat-seq osd.3 2026-03-20T11:21:57.787 INFO:teuthology.orchestra.run.vm02.stdout:47244640259 2026-03-20T11:21:57.787 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd last-stat-seq osd.1 2026-03-20T11:21:57.792 INFO:teuthology.orchestra.run.vm02.stdout:47244640261 2026-03-20T11:21:57.792 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd last-stat-seq osd.0 2026-03-20T11:21:58.008 INFO:teuthology.orchestra.run.vm02.stdout:47244640258 2026-03-20T11:21:58.008 INFO:teuthology.orchestra.run.vm02.stdout:47244640258 2026-03-20T11:21:58.033 INFO:tasks.ceph.ceph_manager.c1:need seq 47244640259 got 47244640258 for osd.5 2026-03-20T11:21:58.035 INFO:tasks.ceph.ceph_manager.c1:need seq 47244640259 got 47244640258 for osd.2 2026-03-20T11:21:58.051 INFO:teuthology.orchestra.run.vm02.stdout:47244640258 2026-03-20T11:21:58.068 INFO:tasks.ceph.ceph_manager.c1:need seq 47244640259 got 47244640258 for osd.4 2026-03-20T11:21:58.094 INFO:teuthology.orchestra.run.vm02.stdout:47244640258 2026-03-20T11:21:58.106 INFO:tasks.ceph.ceph_manager.c1:need seq 47244640259 got 47244640258 for osd.3 2026-03-20T11:21:58.119 INFO:teuthology.orchestra.run.vm02.stdout:47244640258 2026-03-20T11:21:58.133 INFO:tasks.ceph.ceph_manager.c1:need seq 47244640259 got 47244640258 for osd.1 2026-03-20T11:21:58.140 INFO:teuthology.orchestra.run.vm02.stdout:47244640260 2026-03-20T11:21:58.156 INFO:tasks.ceph.ceph_manager.c1:need seq 47244640261 got 47244640260 for osd.0 2026-03-20T11:21:59.034 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd last-stat-seq osd.5 2026-03-20T11:21:59.035 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd last-stat-seq osd.2 2026-03-20T11:21:59.069 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd last-stat-seq osd.4 2026-03-20T11:21:59.107 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd last-stat-seq osd.3 2026-03-20T11:21:59.134 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd last-stat-seq osd.1 2026-03-20T11:21:59.157 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd last-stat-seq osd.0 2026-03-20T11:21:59.267 INFO:teuthology.orchestra.run.vm02.stdout:47244640259 2026-03-20T11:21:59.290 INFO:teuthology.orchestra.run.vm02.stdout:47244640259 2026-03-20T11:21:59.293 INFO:tasks.ceph.ceph_manager.c1:need seq 47244640259 got 47244640259 for osd.2 2026-03-20T11:21:59.293 DEBUG:teuthology.parallel:result is None 2026-03-20T11:21:59.309 INFO:tasks.ceph.ceph_manager.c1:need seq 47244640259 got 47244640259 for osd.5 2026-03-20T11:21:59.309 DEBUG:teuthology.parallel:result is None 2026-03-20T11:21:59.327 INFO:teuthology.orchestra.run.vm02.stdout:47244640259 2026-03-20T11:21:59.345 INFO:tasks.ceph.ceph_manager.c1:need seq 47244640259 got 47244640259 for osd.4 2026-03-20T11:21:59.345 DEBUG:teuthology.parallel:result is None 2026-03-20T11:21:59.408 INFO:teuthology.orchestra.run.vm02.stdout:47244640259 2026-03-20T11:21:59.421 INFO:tasks.ceph.ceph_manager.c1:need seq 47244640259 got 47244640259 for osd.3 2026-03-20T11:21:59.421 DEBUG:teuthology.parallel:result is None 2026-03-20T11:21:59.424 INFO:teuthology.orchestra.run.vm02.stdout:47244640259 2026-03-20T11:21:59.424 INFO:teuthology.orchestra.run.vm02.stdout:47244640261 2026-03-20T11:21:59.436 INFO:tasks.ceph.ceph_manager.c1:need seq 47244640261 got 47244640261 for osd.0 2026-03-20T11:21:59.436 DEBUG:teuthology.parallel:result is None 2026-03-20T11:21:59.436 INFO:tasks.ceph.ceph_manager.c1:need seq 47244640259 got 47244640259 for osd.1 2026-03-20T11:21:59.437 DEBUG:teuthology.parallel:result is None 2026-03-20T11:21:59.437 INFO:tasks.ceph.ceph_manager.c1:waiting for clean 2026-03-20T11:21:59.437 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 pg dump --format=json 2026-03-20T11:21:59.657 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:59.658 INFO:teuthology.orchestra.run.vm02.stderr:dumped all 2026-03-20T11:21:59.668 INFO:teuthology.orchestra.run.vm02.stdout:{"pg_ready":true,"pg_map":{"version":19,"stamp":"2026-03-20T11:21:59.074415+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":191,"num_write_kb":3718,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":155,"ondisk_log_size":155,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":629145600,"kb_used":163312,"kb_used_data":2200,"kb_used_omap":46,"kb_used_meta":160849,"kb_avail":628982288,"statfs":{"total":644245094400,"available":644077862912,"internally_reserved":0,"allocated":2252800,"data_stored":1532850,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48110,"internal_metadata":164709394},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.323305"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":19,"state":"active+clean","last_fresh":"2026-03-20T11:21:56.766919+0000","last_change":"2026-03-20T11:21:56.766996+0000","last_active":"2026-03-20T11:21:56.766919+0000","last_peered":"2026-03-20T11:21:56.766919+0000","last_clean":"2026-03-20T11:21:56.766919+0000","last_became_active":"2026-03-20T11:21:54.754748+0000","last_became_peered":"2026-03-20T11:21:54.754748+0000","last_unstale":"2026-03-20T11:21:56.766919+0000","last_undegraded":"2026-03-20T11:21:56.766919+0000","last_fullsized":"2026-03-20T11:21:56.766919+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_clean_scrub_stamp":"2026-03-20T11:21:53.672667+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T17:21:30.332892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000209423,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":19,"state":"active+clean","last_fresh":"2026-03-20T11:21:56.770316+0000","last_change":"2026-03-20T11:21:56.770395+0000","last_active":"2026-03-20T11:21:56.770316+0000","last_peered":"2026-03-20T11:21:56.770316+0000","last_clean":"2026-03-20T11:21:56.770316+0000","last_became_active":"2026-03-20T11:21:54.755981+0000","last_became_peered":"2026-03-20T11:21:54.755981+0000","last_unstale":"2026-03-20T11:21:56.770316+0000","last_undegraded":"2026-03-20T11:21:56.770316+0000","last_fullsized":"2026-03-20T11:21:56.770316+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_clean_scrub_stamp":"2026-03-20T11:21:53.672667+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T18:34:07.931347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00017813400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":19,"state":"active+clean","last_fresh":"2026-03-20T11:21:56.766934+0000","last_change":"2026-03-20T11:21:56.767010+0000","last_active":"2026-03-20T11:21:56.766934+0000","last_peered":"2026-03-20T11:21:56.766934+0000","last_clean":"2026-03-20T11:21:56.766934+0000","last_became_active":"2026-03-20T11:21:54.865922+0000","last_became_peered":"2026-03-20T11:21:54.865922+0000","last_unstale":"2026-03-20T11:21:56.766934+0000","last_undegraded":"2026-03-20T11:21:56.766934+0000","last_fullsized":"2026-03-20T11:21:56.766934+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_clean_scrub_stamp":"2026-03-20T11:21:53.672667+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T16:25:55.534262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021638500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":19,"state":"active+clean","last_fresh":"2026-03-20T11:21:56.770333+0000","last_change":"2026-03-20T11:21:56.770397+0000","last_active":"2026-03-20T11:21:56.770333+0000","last_peered":"2026-03-20T11:21:56.770333+0000","last_clean":"2026-03-20T11:21:56.770333+0000","last_became_active":"2026-03-20T11:21:54.865069+0000","last_became_peered":"2026-03-20T11:21:54.865069+0000","last_unstale":"2026-03-20T11:21:56.770333+0000","last_undegraded":"2026-03-20T11:21:56.770333+0000","last_fullsized":"2026-03-20T11:21:56.770333+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_clean_scrub_stamp":"2026-03-20T11:21:53.672667+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T18:59:01.203393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000179016,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"19'2","reported_seq":22,"reported_epoch":19,"state":"active+clean","last_fresh":"2026-03-20T11:21:56.769119+0000","last_change":"2026-03-20T11:21:56.769119+0000","last_active":"2026-03-20T11:21:56.769119+0000","last_peered":"2026-03-20T11:21:56.769119+0000","last_clean":"2026-03-20T11:21:56.769119+0000","last_became_active":"2026-03-20T11:21:54.755945+0000","last_became_peered":"2026-03-20T11:21:54.755945+0000","last_unstale":"2026-03-20T11:21:56.769119+0000","last_undegraded":"2026-03-20T11:21:56.769119+0000","last_fullsized":"2026-03-20T11:21:56.769119+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_clean_scrub_stamp":"2026-03-20T11:21:53.672667+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T18:02:38.996309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000465773,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":19,"state":"active+clean","last_fresh":"2026-03-20T11:21:56.786969+0000","last_change":"2026-03-20T11:21:56.787039+0000","last_active":"2026-03-20T11:21:56.786969+0000","last_peered":"2026-03-20T11:21:56.786969+0000","last_clean":"2026-03-20T11:21:56.786969+0000","last_became_active":"2026-03-20T11:21:54.756128+0000","last_became_peered":"2026-03-20T11:21:54.756128+0000","last_unstale":"2026-03-20T11:21:56.786969+0000","last_undegraded":"2026-03-20T11:21:56.786969+0000","last_fullsized":"2026-03-20T11:21:56.786969+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_clean_scrub_stamp":"2026-03-20T11:21:53.672667+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T19:39:41.841031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00015151499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":19,"state":"active+clean","last_fresh":"2026-03-20T11:21:56.767017+0000","last_change":"2026-03-20T11:21:56.767054+0000","last_active":"2026-03-20T11:21:56.767017+0000","last_peered":"2026-03-20T11:21:56.767017+0000","last_clean":"2026-03-20T11:21:56.767017+0000","last_became_active":"2026-03-20T11:21:54.755896+0000","last_became_peered":"2026-03-20T11:21:54.755896+0000","last_unstale":"2026-03-20T11:21:56.767017+0000","last_undegraded":"2026-03-20T11:21:56.767017+0000","last_fullsized":"2026-03-20T11:21:56.767017+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_clean_scrub_stamp":"2026-03-20T11:21:53.672667+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T13:51:07.618211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024065999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"17'1","reported_seq":21,"reported_epoch":19,"state":"active+clean","last_fresh":"2026-03-20T11:21:56.768762+0000","last_change":"2026-03-20T11:21:56.768835+0000","last_active":"2026-03-20T11:21:56.768762+0000","last_peered":"2026-03-20T11:21:56.768762+0000","last_clean":"2026-03-20T11:21:56.768762+0000","last_became_active":"2026-03-20T11:21:54.755061+0000","last_became_peered":"2026-03-20T11:21:54.755061+0000","last_unstale":"2026-03-20T11:21:56.768762+0000","last_undegraded":"2026-03-20T11:21:56.768762+0000","last_fullsized":"2026-03-20T11:21:56.768762+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_clean_scrub_stamp":"2026-03-20T11:21:53.672667+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T14:40:45.386027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000183444,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"14'152","reported_seq":202,"reported_epoch":19,"state":"active+clean","last_fresh":"2026-03-20T11:21:56.766886+0000","last_change":"2026-03-20T11:21:51.845469+0000","last_active":"2026-03-20T11:21:56.766886+0000","last_peered":"2026-03-20T11:21:56.766886+0000","last_clean":"2026-03-20T11:21:56.766886+0000","last_became_active":"2026-03-20T11:21:51.844844+0000","last_became_peered":"2026-03-20T11:21:51.844844+0000","last_unstale":"2026-03-20T11:21:56.766886+0000","last_undegraded":"2026-03-20T11:21:56.766886+0000","last_fullsized":"2026-03-20T11:21:56.766886+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:21:50.584466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:21:50.584466+0000","last_clean_scrub_stamp":"2026-03-20T11:21:50.584466+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T16:04:16.805211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":189,"num_write_kb":3716,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":6},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":189,"num_write_kb":3716,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":152,"ondisk_log_size":152,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":5,"up_from":11,"seq":47244640259,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":27028,"kb_used_data":180,"kb_used_omap":7,"kb_used_meta":26808,"kb_avail":104830572,"statfs":{"total":107374182400,"available":107346505728,"internally_reserved":0,"allocated":184320,"data_stored":60349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8127,"internal_metadata":27451457},"hb_peers":[0,1,2,3,4],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":11,"seq":47244640259,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":27024,"kb_used_data":176,"kb_used_omap":6,"kb_used_meta":26809,"kb_avail":104830576,"statfs":{"total":107374182400,"available":107346509824,"internally_reserved":0,"allocated":180224,"data_stored":60330,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6827,"internal_metadata":27452757},"hb_peers":[0,1,2,3,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":11,"seq":47244640259,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":27596,"kb_used_data":756,"kb_used_omap":8,"kb_used_meta":26807,"kb_avail":104830004,"statfs":{"total":107374182400,"available":107345924096,"internally_reserved":0,"allocated":774144,"data_stored":650698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8775,"internal_metadata":27450809},"hb_peers":[0,1,2,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":11,"seq":47244640259,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":27036,"kb_used_data":164,"kb_used_omap":7,"kb_used_meta":26808,"kb_avail":104830564,"statfs":{"total":107374182400,"available":107346497536,"internally_reserved":0,"allocated":167936,"data_stored":55378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8127,"internal_metadata":27451457},"hb_peers":[0,1,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":11,"seq":47244640259,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":27028,"kb_used_data":180,"kb_used_omap":7,"kb_used_meta":26808,"kb_avail":104830572,"statfs":{"total":107374182400,"available":107346505728,"internally_reserved":0,"allocated":184320,"data_stored":60349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8127,"internal_metadata":27451457},"hb_peers":[0,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":11,"seq":47244640261,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":27600,"kb_used_data":744,"kb_used_omap":7,"kb_used_meta":26808,"kb_avail":104830000,"statfs":{"total":107374182400,"available":107345920000,"internally_reserved":0,"allocated":761856,"data_stored":645746,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8127,"internal_metadata":27451457},"hb_peers":[1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-20T11:21:59.668 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 pg dump --format=json 2026-03-20T11:21:59.854 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:21:59.854 INFO:teuthology.orchestra.run.vm02.stderr:dumped all 2026-03-20T11:21:59.864 INFO:teuthology.orchestra.run.vm02.stdout:{"pg_ready":true,"pg_map":{"version":19,"stamp":"2026-03-20T11:21:59.074415+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":191,"num_write_kb":3718,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":155,"ondisk_log_size":155,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":629145600,"kb_used":163312,"kb_used_data":2200,"kb_used_omap":46,"kb_used_meta":160849,"kb_avail":628982288,"statfs":{"total":644245094400,"available":644077862912,"internally_reserved":0,"allocated":2252800,"data_stored":1532850,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48110,"internal_metadata":164709394},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.323305"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":19,"state":"active+clean","last_fresh":"2026-03-20T11:21:56.766919+0000","last_change":"2026-03-20T11:21:56.766996+0000","last_active":"2026-03-20T11:21:56.766919+0000","last_peered":"2026-03-20T11:21:56.766919+0000","last_clean":"2026-03-20T11:21:56.766919+0000","last_became_active":"2026-03-20T11:21:54.754748+0000","last_became_peered":"2026-03-20T11:21:54.754748+0000","last_unstale":"2026-03-20T11:21:56.766919+0000","last_undegraded":"2026-03-20T11:21:56.766919+0000","last_fullsized":"2026-03-20T11:21:56.766919+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_clean_scrub_stamp":"2026-03-20T11:21:53.672667+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T17:21:30.332892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000209423,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":19,"state":"active+clean","last_fresh":"2026-03-20T11:21:56.770316+0000","last_change":"2026-03-20T11:21:56.770395+0000","last_active":"2026-03-20T11:21:56.770316+0000","last_peered":"2026-03-20T11:21:56.770316+0000","last_clean":"2026-03-20T11:21:56.770316+0000","last_became_active":"2026-03-20T11:21:54.755981+0000","last_became_peered":"2026-03-20T11:21:54.755981+0000","last_unstale":"2026-03-20T11:21:56.770316+0000","last_undegraded":"2026-03-20T11:21:56.770316+0000","last_fullsized":"2026-03-20T11:21:56.770316+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_clean_scrub_stamp":"2026-03-20T11:21:53.672667+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T18:34:07.931347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00017813400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":19,"state":"active+clean","last_fresh":"2026-03-20T11:21:56.766934+0000","last_change":"2026-03-20T11:21:56.767010+0000","last_active":"2026-03-20T11:21:56.766934+0000","last_peered":"2026-03-20T11:21:56.766934+0000","last_clean":"2026-03-20T11:21:56.766934+0000","last_became_active":"2026-03-20T11:21:54.865922+0000","last_became_peered":"2026-03-20T11:21:54.865922+0000","last_unstale":"2026-03-20T11:21:56.766934+0000","last_undegraded":"2026-03-20T11:21:56.766934+0000","last_fullsized":"2026-03-20T11:21:56.766934+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_clean_scrub_stamp":"2026-03-20T11:21:53.672667+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T16:25:55.534262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021638500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":19,"state":"active+clean","last_fresh":"2026-03-20T11:21:56.770333+0000","last_change":"2026-03-20T11:21:56.770397+0000","last_active":"2026-03-20T11:21:56.770333+0000","last_peered":"2026-03-20T11:21:56.770333+0000","last_clean":"2026-03-20T11:21:56.770333+0000","last_became_active":"2026-03-20T11:21:54.865069+0000","last_became_peered":"2026-03-20T11:21:54.865069+0000","last_unstale":"2026-03-20T11:21:56.770333+0000","last_undegraded":"2026-03-20T11:21:56.770333+0000","last_fullsized":"2026-03-20T11:21:56.770333+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_clean_scrub_stamp":"2026-03-20T11:21:53.672667+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T18:59:01.203393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000179016,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"19'2","reported_seq":22,"reported_epoch":19,"state":"active+clean","last_fresh":"2026-03-20T11:21:56.769119+0000","last_change":"2026-03-20T11:21:56.769119+0000","last_active":"2026-03-20T11:21:56.769119+0000","last_peered":"2026-03-20T11:21:56.769119+0000","last_clean":"2026-03-20T11:21:56.769119+0000","last_became_active":"2026-03-20T11:21:54.755945+0000","last_became_peered":"2026-03-20T11:21:54.755945+0000","last_unstale":"2026-03-20T11:21:56.769119+0000","last_undegraded":"2026-03-20T11:21:56.769119+0000","last_fullsized":"2026-03-20T11:21:56.769119+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_clean_scrub_stamp":"2026-03-20T11:21:53.672667+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T18:02:38.996309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000465773,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":19,"state":"active+clean","last_fresh":"2026-03-20T11:21:56.786969+0000","last_change":"2026-03-20T11:21:56.787039+0000","last_active":"2026-03-20T11:21:56.786969+0000","last_peered":"2026-03-20T11:21:56.786969+0000","last_clean":"2026-03-20T11:21:56.786969+0000","last_became_active":"2026-03-20T11:21:54.756128+0000","last_became_peered":"2026-03-20T11:21:54.756128+0000","last_unstale":"2026-03-20T11:21:56.786969+0000","last_undegraded":"2026-03-20T11:21:56.786969+0000","last_fullsized":"2026-03-20T11:21:56.786969+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_clean_scrub_stamp":"2026-03-20T11:21:53.672667+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T19:39:41.841031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00015151499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":19,"state":"active+clean","last_fresh":"2026-03-20T11:21:56.767017+0000","last_change":"2026-03-20T11:21:56.767054+0000","last_active":"2026-03-20T11:21:56.767017+0000","last_peered":"2026-03-20T11:21:56.767017+0000","last_clean":"2026-03-20T11:21:56.767017+0000","last_became_active":"2026-03-20T11:21:54.755896+0000","last_became_peered":"2026-03-20T11:21:54.755896+0000","last_unstale":"2026-03-20T11:21:56.767017+0000","last_undegraded":"2026-03-20T11:21:56.767017+0000","last_fullsized":"2026-03-20T11:21:56.767017+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_clean_scrub_stamp":"2026-03-20T11:21:53.672667+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T13:51:07.618211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024065999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"17'1","reported_seq":21,"reported_epoch":19,"state":"active+clean","last_fresh":"2026-03-20T11:21:56.768762+0000","last_change":"2026-03-20T11:21:56.768835+0000","last_active":"2026-03-20T11:21:56.768762+0000","last_peered":"2026-03-20T11:21:56.768762+0000","last_clean":"2026-03-20T11:21:56.768762+0000","last_became_active":"2026-03-20T11:21:54.755061+0000","last_became_peered":"2026-03-20T11:21:54.755061+0000","last_unstale":"2026-03-20T11:21:56.768762+0000","last_undegraded":"2026-03-20T11:21:56.768762+0000","last_fullsized":"2026-03-20T11:21:56.768762+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:21:53.672667+0000","last_clean_scrub_stamp":"2026-03-20T11:21:53.672667+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T14:40:45.386027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000183444,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"14'152","reported_seq":202,"reported_epoch":19,"state":"active+clean","last_fresh":"2026-03-20T11:21:56.766886+0000","last_change":"2026-03-20T11:21:51.845469+0000","last_active":"2026-03-20T11:21:56.766886+0000","last_peered":"2026-03-20T11:21:56.766886+0000","last_clean":"2026-03-20T11:21:56.766886+0000","last_became_active":"2026-03-20T11:21:51.844844+0000","last_became_peered":"2026-03-20T11:21:51.844844+0000","last_unstale":"2026-03-20T11:21:56.766886+0000","last_undegraded":"2026-03-20T11:21:56.766886+0000","last_fullsized":"2026-03-20T11:21:56.766886+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:21:50.584466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:21:50.584466+0000","last_clean_scrub_stamp":"2026-03-20T11:21:50.584466+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T16:04:16.805211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":189,"num_write_kb":3716,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":6},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":189,"num_write_kb":3716,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":152,"ondisk_log_size":152,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":5,"up_from":11,"seq":47244640259,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":27028,"kb_used_data":180,"kb_used_omap":7,"kb_used_meta":26808,"kb_avail":104830572,"statfs":{"total":107374182400,"available":107346505728,"internally_reserved":0,"allocated":184320,"data_stored":60349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8127,"internal_metadata":27451457},"hb_peers":[0,1,2,3,4],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":11,"seq":47244640259,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":27024,"kb_used_data":176,"kb_used_omap":6,"kb_used_meta":26809,"kb_avail":104830576,"statfs":{"total":107374182400,"available":107346509824,"internally_reserved":0,"allocated":180224,"data_stored":60330,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6827,"internal_metadata":27452757},"hb_peers":[0,1,2,3,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":11,"seq":47244640259,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":27596,"kb_used_data":756,"kb_used_omap":8,"kb_used_meta":26807,"kb_avail":104830004,"statfs":{"total":107374182400,"available":107345924096,"internally_reserved":0,"allocated":774144,"data_stored":650698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8775,"internal_metadata":27450809},"hb_peers":[0,1,2,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":11,"seq":47244640259,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":27036,"kb_used_data":164,"kb_used_omap":7,"kb_used_meta":26808,"kb_avail":104830564,"statfs":{"total":107374182400,"available":107346497536,"internally_reserved":0,"allocated":167936,"data_stored":55378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8127,"internal_metadata":27451457},"hb_peers":[0,1,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":11,"seq":47244640259,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":27028,"kb_used_data":180,"kb_used_omap":7,"kb_used_meta":26808,"kb_avail":104830572,"statfs":{"total":107374182400,"available":107346505728,"internally_reserved":0,"allocated":184320,"data_stored":60349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8127,"internal_metadata":27451457},"hb_peers":[0,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":11,"seq":47244640261,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":27600,"kb_used_data":744,"kb_used_omap":7,"kb_used_meta":26808,"kb_avail":104830000,"statfs":{"total":107374182400,"available":107345920000,"internally_reserved":0,"allocated":761856,"data_stored":645746,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8127,"internal_metadata":27451457},"hb_peers":[1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-20T11:21:59.865 INFO:tasks.ceph.ceph_manager.c1:clean! 2026-03-20T11:21:59.865 INFO:tasks.ceph:Waiting until ceph cluster c1 is healthy... 2026-03-20T11:21:59.865 INFO:tasks.ceph.ceph_manager.c1:wait_until_healthy 2026-03-20T11:21:59.865 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 health --format=json 2026-03-20T11:22:00.083 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:00.083 INFO:teuthology.orchestra.run.vm02.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-03-20T11:22:00.095 INFO:tasks.ceph.ceph_manager.c1:wait_until_healthy done 2026-03-20T11:22:00.095 INFO:teuthology.run_tasks:Running task ceph... 2026-03-20T11:22:00.097 INFO:tasks.ceph:Creating ceph cluster c2... 2026-03-20T11:22:00.097 INFO:tasks.ceph:config {'conf': {'client': {'debug rgw': 20, 'debug rgw lifecycle': 20, 'debug rgw notification': 20, 'debug rgw sync': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=', 'rgw curl low speed time': 300, 'rgw data log num shards': 4, 'rgw data sync poll interval': 5, 'rgw md log max shards': 4, 'rgw meta sync poll interval': 5, 'rgw s3 auth use sts': True, 'rgw sts key': 'abcdefghijklmnoq', 'rgw sync data inject err probability': 0, 'rgw sync log trim interval': 0, 'rgw sync meta inject err probability': 0, 'rgw sync obj etag verify': True, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'client.0': {'rgw data notify interval msec': 0}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd mclock iops capacity threshold hdd': 49000, 'rocksdb delete range threshold': 0}, 'osd.0': {'osd_max_omap_entries_per_request': 10}, 'osd.1': {'osd_max_omap_entries_per_request': 1000}, 'osd.2': {'osd_max_omap_entries_per_request': 10000}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(PG_AVAILABILITY\\)', '\\(PG_DEGRADED\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'not have an application enabled'], 'cpu_profile': set(), 'cluster': 'c2', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-03-20T11:22:00.097 INFO:tasks.ceph:ctx.config {'archive_path': '/archive/kyr-2026-03-20_10:58:43-rgw-tentacle-none-default-vps/2068', 'branch': 'tentacle', 'description': 'rgw/multisite/{clusters frontend/beast ignore-pg-availability notify omap_limits overrides realms/two-zonegroup supported-random-distro$/{centos_latest} tasks/test_multi}', 'email': None, 'first_in_suite': False, 'flavor': 'default', 'job_id': '2068', 'last_in_suite': False, 'machine_type': 'vps', 'name': 'kyr-2026-03-20_10:58:43-rgw-tentacle-none-default-vps', 'no_nested_subset': False, 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'tentacle'}, 'ansible.cephlab': {'branch': 'main', 'repo': 'https://github.com/kshtsk/ceph-cm-ansible.git', 'skip_tags': 'nagios,monitoring-scripts,hostname,pubkeys,zap,sudoers,kerberos,ntp-client,resolvconf,cpan,nfs', 'vars': {'logical_volumes': {'lv_1': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}, 'lv_2': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}, 'lv_3': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}, 'lv_4': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}}, 'timezone': 'UTC', 'volume_groups': {'vg_nvme': {'pvs': '/dev/vdb,/dev/vdc,/dev/vdd,/dev/vde'}}}}, 'ceph': {'conf': {'client': {'debug rgw': 20, 'debug rgw lifecycle': 20, 'debug rgw notification': 20, 'debug rgw sync': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=', 'rgw curl low speed time': 300, 'rgw data log num shards': 4, 'rgw data sync poll interval': 5, 'rgw md log max shards': 4, 'rgw meta sync poll interval': 5, 'rgw s3 auth use sts': True, 'rgw sts key': 'abcdefghijklmnoq', 'rgw sync data inject err probability': 0, 'rgw sync log trim interval': 0, 'rgw sync meta inject err probability': 0, 'rgw sync obj etag verify': True, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'client.0': {'rgw data notify interval msec': 0}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd mclock iops capacity threshold hdd': 49000, 'rocksdb delete range threshold': 0}, 'osd.0': {'osd_max_omap_entries_per_request': 10}, 'osd.1': {'osd_max_omap_entries_per_request': 1000}, 'osd.2': {'osd_max_omap_entries_per_request': 10000}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(PG_AVAILABILITY\\)', '\\(PG_DEGRADED\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'not have an application enabled'], 'sha1': '70f8415b300f041766fa27faf7d5472699e32388'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'cephadm': {'cephadm_binary_url': 'https://download.ceph.com/rpm-20.2.0/el9/noarch/cephadm'}, 'install': {'ceph': {'flavor': 'default', 'sha1': '70f8415b300f041766fa27faf7d5472699e32388'}, 'extra_system_packages': {'deb': ['python3-jmespath', 'python3-xmltodict', 's3cmd'], 'rpm': ['bzip2', 'perl-Test-Harness', 'python3-jmespath', 'python3-xmltodict', 's3cmd']}}, 'rgw': {'compression type': 'random', 'frontend': 'beast'}, 'rgw-multisite': {'realm': {'is_default': True, 'name': 'test-realm'}, 'zonegroups': [{'enabled_features': ['resharding', 'notification_v2'], 'endpoints': ['c1.client.0'], 'is_default': True, 'is_master': True, 'name': 'a', 'zones': [{'endpoints': ['c1.client.0'], 'is_default': True, 'is_master': True, 'name': 'a1'}, {'endpoints': ['c1.client.1'], 'name': 'a2'}]}, {'enabled_features': ['resharding', 'notification_v2'], 'endpoints': ['c2.client.0'], 'is_default': True, 'name': 'b', 'zones': [{'endpoints': ['c2.client.0'], 'is_default': True, 'is_master': True, 'name': 'b1'}, {'endpoints': ['c2.client.1'], 'name': 'b2'}]}]}, 'rgw-multisite-tests': {'args': ['tests.py', '-a', '!fails_with_rgw']}, 'workunit': {'branch': 'tt-tentacle', 'sha1': '7b4fb1902b22ff4ea3f4ff6a953bc42198ebeffe'}}, 'owner': 'kyr', 'priority': 1000, 'repo': 'https://github.com/ceph/ceph.git', 'roles': [['c1.mon.a', 'c1.osd.0', 'c1.osd.1', 'c1.osd.2', 'c1.client.0'], ['c1.mgr.x', 'c1.osd.3', 'c1.osd.4', 'c1.osd.5', 'c1.client.1'], ['c2.mon.a', 'c2.osd.0', 'c2.osd.1', 'c2.osd.2', 'c2.client.0'], ['c2.mgr.x', 'c2.osd.3', 'c2.osd.4', 'c2.osd.5', 'c2.client.1']], 'seed': 7702, 'sha1': '70f8415b300f041766fa27faf7d5472699e32388', 'sleep_before_teardown': 0, 'suite': 'rgw', 'suite_branch': 'tt-tentacle', 'suite_path': '/home/teuthos/src/github.com_kshtsk_ceph_7b4fb1902b22ff4ea3f4ff6a953bc42198ebeffe/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/kshtsk/ceph.git', 'suite_sha1': '7b4fb1902b22ff4ea3f4ff6a953bc42198ebeffe', 'targets': {'vm02.local': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHDOdGzmlLQ/ZTYxKoEn2ygRbjgX4ux/lNWXbj7ocH2unOqJgDf8+OP1BWuHYNEv7h7psNGXZ0ZkhElY2KRateg=', 'vm05.local': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIs+bAjIBCO2Z1GDSS25TON+TMyqw5NQcnquTRycytbaJNkFJi5yDkOkkdeCgrmVAOljPOfw/yGvTfVtaKsCVhw=', 'vm07.local': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDlMCt1zzzTlX2IlgevpvFzoeoxOiYD4Qk8mFXF1w3snFJ7povvBC+RimEI07rEIHxU3hlLnvg+Tit+Yf5M5DAs=', 'vm08.local': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNug/VtQYdbJ4Dz5jL/XJ5LvMMhWwcvBq6Amz05ZwPG64TWDh0niEw8yvtwAlDN8ZBITlJN1keEDVaud9T22nD4='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': None}, {'ceph': {'cluster': 'c1', 'conf': {'client': {'debug rgw': 20, 'debug rgw lifecycle': 20, 'debug rgw notification': 20, 'debug rgw sync': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=', 'rgw curl low speed time': 300, 'rgw data log num shards': 4, 'rgw data sync poll interval': 5, 'rgw md log max shards': 4, 'rgw meta sync poll interval': 5, 'rgw s3 auth use sts': True, 'rgw sts key': 'abcdefghijklmnoq', 'rgw sync data inject err probability': 0, 'rgw sync log trim interval': 0, 'rgw sync meta inject err probability': 0, 'rgw sync obj etag verify': True, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'client.0': {'rgw data notify interval msec': 0}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd mclock iops capacity threshold hdd': 49000, 'rocksdb delete range threshold': 0}, 'osd.0': {'osd_max_omap_entries_per_request': 10}, 'osd.1': {'osd_max_omap_entries_per_request': 1000}, 'osd.2': {'osd_max_omap_entries_per_request': 10000}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(PG_AVAILABILITY\\)', '\\(PG_DEGRADED\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'not have an application enabled'], 'sha1': '70f8415b300f041766fa27faf7d5472699e32388'}}, {'ceph': {'cluster': 'c2', 'conf': {'client': {'debug rgw': 20, 'debug rgw lifecycle': 20, 'debug rgw notification': 20, 'debug rgw sync': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=', 'rgw curl low speed time': 300, 'rgw data log num shards': 4, 'rgw data sync poll interval': 5, 'rgw md log max shards': 4, 'rgw meta sync poll interval': 5, 'rgw s3 auth use sts': True, 'rgw sts key': 'abcdefghijklmnoq', 'rgw sync data inject err probability': 0, 'rgw sync log trim interval': 0, 'rgw sync meta inject err probability': 0, 'rgw sync obj etag verify': True, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'client.0': {'rgw data notify interval msec': 0}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd mclock iops capacity threshold hdd': 49000, 'rocksdb delete range threshold': 0}, 'osd.0': {'osd_max_omap_entries_per_request': 10}, 'osd.1': {'osd_max_omap_entries_per_request': 1000}, 'osd.2': {'osd_max_omap_entries_per_request': 10000}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(PG_AVAILABILITY\\)', '\\(PG_DEGRADED\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'not have an application enabled'], 'sha1': '70f8415b300f041766fa27faf7d5472699e32388'}}, {'rgw': {'c1.client.0': {'port': 8000}, 'c1.client.1': {'port': 8001}, 'c2.client.0': {'port': 8000}, 'c2.client.1': {'port': 8001}}}, {'rgw-multisite': None}, {'rgw-multisite-tests': {'config': {'reconfigure_delay': 90}}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'clyso-debian-13', 'teuthology_repo': 'https://github.com/clyso/teuthology', 'teuthology_sha1': '1c580df7a9c7c2aadc272da296344fd99f27c444', 'timestamp': '2026-03-20_10:58:43', 'tube': 'vps', 'user': 'kyr', 'verbose': False, 'worker_log': '/home/teuthos/.teuthology/dispatcher/dispatcher.vps.4188345'} 2026-03-20T11:22:00.097 DEBUG:teuthology.orchestra.run.vm02:> install -d -m0755 -- /home/ubuntu/cephtest/c2.data 2026-03-20T11:22:00.099 DEBUG:teuthology.orchestra.run.vm05:> install -d -m0755 -- /home/ubuntu/cephtest/c2.data 2026-03-20T11:22:00.100 DEBUG:teuthology.orchestra.run.vm07:> install -d -m0755 -- /home/ubuntu/cephtest/c2.data 2026-03-20T11:22:00.102 DEBUG:teuthology.orchestra.run.vm08:> install -d -m0755 -- /home/ubuntu/cephtest/c2.data 2026-03-20T11:22:00.118 DEBUG:teuthology.orchestra.run.vm02:> sudo install -d -m0777 -- /var/run/ceph 2026-03-20T11:22:00.153 DEBUG:teuthology.orchestra.run.vm05:> sudo install -d -m0777 -- /var/run/ceph 2026-03-20T11:22:00.157 DEBUG:teuthology.orchestra.run.vm07:> sudo install -d -m0777 -- /var/run/ceph 2026-03-20T11:22:00.160 DEBUG:teuthology.orchestra.run.vm08:> sudo install -d -m0777 -- /var/run/ceph 2026-03-20T11:22:00.185 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-20T11:22:00.185 DEBUG:teuthology.orchestra.run.vm07:> dd if=/scratch_devs of=/dev/stdout 2026-03-20T11:22:00.242 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-03-20T11:22:00.243 DEBUG:teuthology.orchestra.run.vm07:> stat /dev/vg_nvme/lv_1 2026-03-20T11:22:00.300 INFO:teuthology.orchestra.run.vm07.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-03-20T11:22:00.300 INFO:teuthology.orchestra.run.vm07.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-20T11:22:00.300 INFO:teuthology.orchestra.run.vm07.stdout:Device: 6h/6d Inode: 631 Links: 1 2026-03-20T11:22:00.300 INFO:teuthology.orchestra.run.vm07.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-20T11:22:00.300 INFO:teuthology.orchestra.run.vm07.stdout:Context: system_u:object_r:device_t:s0 2026-03-20T11:22:00.300 INFO:teuthology.orchestra.run.vm07.stdout:Access: 2026-03-20 11:21:15.661658697 +0000 2026-03-20T11:22:00.300 INFO:teuthology.orchestra.run.vm07.stdout:Modify: 2026-03-20 11:19:20.814243144 +0000 2026-03-20T11:22:00.300 INFO:teuthology.orchestra.run.vm07.stdout:Change: 2026-03-20 11:19:20.814243144 +0000 2026-03-20T11:22:00.300 INFO:teuthology.orchestra.run.vm07.stdout: Birth: 2026-03-20 11:19:20.814243144 +0000 2026-03-20T11:22:00.300 DEBUG:teuthology.orchestra.run.vm07:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-03-20T11:22:00.364 INFO:teuthology.orchestra.run.vm07.stderr:1+0 records in 2026-03-20T11:22:00.364 INFO:teuthology.orchestra.run.vm07.stderr:1+0 records out 2026-03-20T11:22:00.364 INFO:teuthology.orchestra.run.vm07.stderr:512 bytes copied, 0.000137228 s, 3.7 MB/s 2026-03-20T11:22:00.365 DEBUG:teuthology.orchestra.run.vm07:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-03-20T11:22:00.421 DEBUG:teuthology.orchestra.run.vm07:> stat /dev/vg_nvme/lv_2 2026-03-20T11:22:00.478 INFO:teuthology.orchestra.run.vm07.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-03-20T11:22:00.478 INFO:teuthology.orchestra.run.vm07.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-20T11:22:00.478 INFO:teuthology.orchestra.run.vm07.stdout:Device: 6h/6d Inode: 683 Links: 1 2026-03-20T11:22:00.478 INFO:teuthology.orchestra.run.vm07.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-20T11:22:00.478 INFO:teuthology.orchestra.run.vm07.stdout:Context: system_u:object_r:device_t:s0 2026-03-20T11:22:00.478 INFO:teuthology.orchestra.run.vm07.stdout:Access: 2026-03-20 11:21:15.661658697 +0000 2026-03-20T11:22:00.478 INFO:teuthology.orchestra.run.vm07.stdout:Modify: 2026-03-20 11:19:21.088243398 +0000 2026-03-20T11:22:00.478 INFO:teuthology.orchestra.run.vm07.stdout:Change: 2026-03-20 11:19:21.088243398 +0000 2026-03-20T11:22:00.478 INFO:teuthology.orchestra.run.vm07.stdout: Birth: 2026-03-20 11:19:21.088243398 +0000 2026-03-20T11:22:00.478 DEBUG:teuthology.orchestra.run.vm07:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-03-20T11:22:00.541 INFO:teuthology.orchestra.run.vm07.stderr:1+0 records in 2026-03-20T11:22:00.541 INFO:teuthology.orchestra.run.vm07.stderr:1+0 records out 2026-03-20T11:22:00.541 INFO:teuthology.orchestra.run.vm07.stderr:512 bytes copied, 0.000136877 s, 3.7 MB/s 2026-03-20T11:22:00.542 DEBUG:teuthology.orchestra.run.vm07:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-03-20T11:22:00.599 DEBUG:teuthology.orchestra.run.vm07:> stat /dev/vg_nvme/lv_3 2026-03-20T11:22:00.655 INFO:teuthology.orchestra.run.vm07.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-03-20T11:22:00.655 INFO:teuthology.orchestra.run.vm07.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-20T11:22:00.655 INFO:teuthology.orchestra.run.vm07.stdout:Device: 6h/6d Inode: 720 Links: 1 2026-03-20T11:22:00.655 INFO:teuthology.orchestra.run.vm07.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-20T11:22:00.655 INFO:teuthology.orchestra.run.vm07.stdout:Context: system_u:object_r:device_t:s0 2026-03-20T11:22:00.655 INFO:teuthology.orchestra.run.vm07.stdout:Access: 2026-03-20 11:21:15.661658697 +0000 2026-03-20T11:22:00.655 INFO:teuthology.orchestra.run.vm07.stdout:Modify: 2026-03-20 11:19:21.355243646 +0000 2026-03-20T11:22:00.655 INFO:teuthology.orchestra.run.vm07.stdout:Change: 2026-03-20 11:19:21.355243646 +0000 2026-03-20T11:22:00.655 INFO:teuthology.orchestra.run.vm07.stdout: Birth: 2026-03-20 11:19:21.355243646 +0000 2026-03-20T11:22:00.656 DEBUG:teuthology.orchestra.run.vm07:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-03-20T11:22:00.719 INFO:teuthology.orchestra.run.vm07.stderr:1+0 records in 2026-03-20T11:22:00.719 INFO:teuthology.orchestra.run.vm07.stderr:1+0 records out 2026-03-20T11:22:00.719 INFO:teuthology.orchestra.run.vm07.stderr:512 bytes copied, 0.000139991 s, 3.7 MB/s 2026-03-20T11:22:00.720 DEBUG:teuthology.orchestra.run.vm07:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-03-20T11:22:00.777 DEBUG:teuthology.orchestra.run.vm07:> stat /dev/vg_nvme/lv_4 2026-03-20T11:22:00.835 INFO:teuthology.orchestra.run.vm07.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-03-20T11:22:00.835 INFO:teuthology.orchestra.run.vm07.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-20T11:22:00.835 INFO:teuthology.orchestra.run.vm07.stdout:Device: 6h/6d Inode: 756 Links: 1 2026-03-20T11:22:00.835 INFO:teuthology.orchestra.run.vm07.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-20T11:22:00.835 INFO:teuthology.orchestra.run.vm07.stdout:Context: system_u:object_r:device_t:s0 2026-03-20T11:22:00.835 INFO:teuthology.orchestra.run.vm07.stdout:Access: 2026-03-20 11:21:15.662658699 +0000 2026-03-20T11:22:00.835 INFO:teuthology.orchestra.run.vm07.stdout:Modify: 2026-03-20 11:19:21.624243897 +0000 2026-03-20T11:22:00.835 INFO:teuthology.orchestra.run.vm07.stdout:Change: 2026-03-20 11:19:21.624243897 +0000 2026-03-20T11:22:00.835 INFO:teuthology.orchestra.run.vm07.stdout: Birth: 2026-03-20 11:19:21.624243897 +0000 2026-03-20T11:22:00.835 DEBUG:teuthology.orchestra.run.vm07:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-03-20T11:22:00.902 INFO:teuthology.orchestra.run.vm07.stderr:1+0 records in 2026-03-20T11:22:00.902 INFO:teuthology.orchestra.run.vm07.stderr:1+0 records out 2026-03-20T11:22:00.902 INFO:teuthology.orchestra.run.vm07.stderr:512 bytes copied, 0.000152356 s, 3.4 MB/s 2026-03-20T11:22:00.903 DEBUG:teuthology.orchestra.run.vm07:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-03-20T11:22:00.964 INFO:tasks.ceph:osd dev map: {'c2.osd.0': '/dev/vg_nvme/lv_1', 'c2.osd.1': '/dev/vg_nvme/lv_2', 'c2.osd.2': '/dev/vg_nvme/lv_3'} 2026-03-20T11:22:00.964 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-20T11:22:00.964 DEBUG:teuthology.orchestra.run.vm08:> dd if=/scratch_devs of=/dev/stdout 2026-03-20T11:22:00.982 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-03-20T11:22:00.982 DEBUG:teuthology.orchestra.run.vm08:> stat /dev/vg_nvme/lv_1 2026-03-20T11:22:01.039 INFO:teuthology.orchestra.run.vm08.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-03-20T11:22:01.039 INFO:teuthology.orchestra.run.vm08.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-20T11:22:01.039 INFO:teuthology.orchestra.run.vm08.stdout:Device: 6h/6d Inode: 661 Links: 1 2026-03-20T11:22:01.039 INFO:teuthology.orchestra.run.vm08.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-20T11:22:01.039 INFO:teuthology.orchestra.run.vm08.stdout:Context: system_u:object_r:device_t:s0 2026-03-20T11:22:01.039 INFO:teuthology.orchestra.run.vm08.stdout:Access: 2026-03-20 11:21:29.171326236 +0000 2026-03-20T11:22:01.039 INFO:teuthology.orchestra.run.vm08.stdout:Modify: 2026-03-20 11:19:25.159831286 +0000 2026-03-20T11:22:01.039 INFO:teuthology.orchestra.run.vm08.stdout:Change: 2026-03-20 11:19:25.159831286 +0000 2026-03-20T11:22:01.039 INFO:teuthology.orchestra.run.vm08.stdout: Birth: 2026-03-20 11:19:25.159831286 +0000 2026-03-20T11:22:01.040 DEBUG:teuthology.orchestra.run.vm08:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-03-20T11:22:01.103 INFO:teuthology.orchestra.run.vm08.stderr:1+0 records in 2026-03-20T11:22:01.103 INFO:teuthology.orchestra.run.vm08.stderr:1+0 records out 2026-03-20T11:22:01.103 INFO:teuthology.orchestra.run.vm08.stderr:512 bytes copied, 0.000163987 s, 3.1 MB/s 2026-03-20T11:22:01.104 DEBUG:teuthology.orchestra.run.vm08:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-03-20T11:22:01.158 DEBUG:teuthology.orchestra.run.vm08:> stat /dev/vg_nvme/lv_2 2026-03-20T11:22:01.216 INFO:teuthology.orchestra.run.vm08.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-03-20T11:22:01.216 INFO:teuthology.orchestra.run.vm08.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-20T11:22:01.216 INFO:teuthology.orchestra.run.vm08.stdout:Device: 6h/6d Inode: 690 Links: 1 2026-03-20T11:22:01.216 INFO:teuthology.orchestra.run.vm08.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-20T11:22:01.216 INFO:teuthology.orchestra.run.vm08.stdout:Context: system_u:object_r:device_t:s0 2026-03-20T11:22:01.216 INFO:teuthology.orchestra.run.vm08.stdout:Access: 2026-03-20 11:21:29.171326236 +0000 2026-03-20T11:22:01.216 INFO:teuthology.orchestra.run.vm08.stdout:Modify: 2026-03-20 11:19:25.407831837 +0000 2026-03-20T11:22:01.216 INFO:teuthology.orchestra.run.vm08.stdout:Change: 2026-03-20 11:19:25.407831837 +0000 2026-03-20T11:22:01.216 INFO:teuthology.orchestra.run.vm08.stdout: Birth: 2026-03-20 11:19:25.407831837 +0000 2026-03-20T11:22:01.216 DEBUG:teuthology.orchestra.run.vm08:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-03-20T11:22:01.280 INFO:teuthology.orchestra.run.vm08.stderr:1+0 records in 2026-03-20T11:22:01.280 INFO:teuthology.orchestra.run.vm08.stderr:1+0 records out 2026-03-20T11:22:01.280 INFO:teuthology.orchestra.run.vm08.stderr:512 bytes copied, 0.000130584 s, 3.9 MB/s 2026-03-20T11:22:01.281 DEBUG:teuthology.orchestra.run.vm08:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-03-20T11:22:01.338 DEBUG:teuthology.orchestra.run.vm08:> stat /dev/vg_nvme/lv_3 2026-03-20T11:22:01.398 INFO:teuthology.orchestra.run.vm08.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-03-20T11:22:01.398 INFO:teuthology.orchestra.run.vm08.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-20T11:22:01.398 INFO:teuthology.orchestra.run.vm08.stdout:Device: 6h/6d Inode: 711 Links: 1 2026-03-20T11:22:01.398 INFO:teuthology.orchestra.run.vm08.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-20T11:22:01.398 INFO:teuthology.orchestra.run.vm08.stdout:Context: system_u:object_r:device_t:s0 2026-03-20T11:22:01.398 INFO:teuthology.orchestra.run.vm08.stdout:Access: 2026-03-20 11:21:29.172326238 +0000 2026-03-20T11:22:01.398 INFO:teuthology.orchestra.run.vm08.stdout:Modify: 2026-03-20 11:19:25.661832400 +0000 2026-03-20T11:22:01.398 INFO:teuthology.orchestra.run.vm08.stdout:Change: 2026-03-20 11:19:25.661832400 +0000 2026-03-20T11:22:01.398 INFO:teuthology.orchestra.run.vm08.stdout: Birth: 2026-03-20 11:19:25.661832400 +0000 2026-03-20T11:22:01.398 DEBUG:teuthology.orchestra.run.vm08:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-03-20T11:22:01.465 INFO:teuthology.orchestra.run.vm08.stderr:1+0 records in 2026-03-20T11:22:01.465 INFO:teuthology.orchestra.run.vm08.stderr:1+0 records out 2026-03-20T11:22:01.465 INFO:teuthology.orchestra.run.vm08.stderr:512 bytes copied, 0.000115626 s, 4.4 MB/s 2026-03-20T11:22:01.466 DEBUG:teuthology.orchestra.run.vm08:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-03-20T11:22:01.524 DEBUG:teuthology.orchestra.run.vm08:> stat /dev/vg_nvme/lv_4 2026-03-20T11:22:01.583 INFO:teuthology.orchestra.run.vm08.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-03-20T11:22:01.583 INFO:teuthology.orchestra.run.vm08.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-20T11:22:01.583 INFO:teuthology.orchestra.run.vm08.stdout:Device: 6h/6d Inode: 751 Links: 1 2026-03-20T11:22:01.583 INFO:teuthology.orchestra.run.vm08.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-20T11:22:01.583 INFO:teuthology.orchestra.run.vm08.stdout:Context: system_u:object_r:device_t:s0 2026-03-20T11:22:01.583 INFO:teuthology.orchestra.run.vm08.stdout:Access: 2026-03-20 11:21:29.172326238 +0000 2026-03-20T11:22:01.583 INFO:teuthology.orchestra.run.vm08.stdout:Modify: 2026-03-20 11:19:25.934833005 +0000 2026-03-20T11:22:01.583 INFO:teuthology.orchestra.run.vm08.stdout:Change: 2026-03-20 11:19:25.934833005 +0000 2026-03-20T11:22:01.583 INFO:teuthology.orchestra.run.vm08.stdout: Birth: 2026-03-20 11:19:25.934833005 +0000 2026-03-20T11:22:01.584 DEBUG:teuthology.orchestra.run.vm08:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-03-20T11:22:01.651 INFO:teuthology.orchestra.run.vm08.stderr:1+0 records in 2026-03-20T11:22:01.651 INFO:teuthology.orchestra.run.vm08.stderr:1+0 records out 2026-03-20T11:22:01.651 INFO:teuthology.orchestra.run.vm08.stderr:512 bytes copied, 0.000148587 s, 3.4 MB/s 2026-03-20T11:22:01.653 DEBUG:teuthology.orchestra.run.vm08:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-03-20T11:22:01.710 INFO:tasks.ceph:osd dev map: {'c2.osd.3': '/dev/vg_nvme/lv_1', 'c2.osd.4': '/dev/vg_nvme/lv_2', 'c2.osd.5': '/dev/vg_nvme/lv_3'} 2026-03-20T11:22:01.710 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@vm07.local'): {'c2.osd.0': '/dev/vg_nvme/lv_1', 'c2.osd.1': '/dev/vg_nvme/lv_2', 'c2.osd.2': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@vm08.local'): {'c2.osd.3': '/dev/vg_nvme/lv_1', 'c2.osd.4': '/dev/vg_nvme/lv_2', 'c2.osd.5': '/dev/vg_nvme/lv_3'}} 2026-03-20T11:22:01.710 INFO:tasks.ceph:Generating config... 2026-03-20T11:22:01.711 INFO:tasks.ceph:[client] debug rgw = 20 2026-03-20T11:22:01.711 INFO:tasks.ceph:[client] debug rgw lifecycle = 20 2026-03-20T11:22:01.711 INFO:tasks.ceph:[client] debug rgw notification = 20 2026-03-20T11:22:01.711 INFO:tasks.ceph:[client] debug rgw sync = 20 2026-03-20T11:22:01.711 INFO:tasks.ceph:[client] rgw crypt require ssl = False 2026-03-20T11:22:01.711 INFO:tasks.ceph:[client] rgw crypt s3 kms backend = testing 2026-03-20T11:22:01.711 INFO:tasks.ceph:[client] rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= 2026-03-20T11:22:01.711 INFO:tasks.ceph:[client] rgw curl low speed time = 300 2026-03-20T11:22:01.711 INFO:tasks.ceph:[client] rgw data log num shards = 4 2026-03-20T11:22:01.711 INFO:tasks.ceph:[client] rgw data sync poll interval = 5 2026-03-20T11:22:01.711 INFO:tasks.ceph:[client] rgw md log max shards = 4 2026-03-20T11:22:01.711 INFO:tasks.ceph:[client] rgw meta sync poll interval = 5 2026-03-20T11:22:01.711 INFO:tasks.ceph:[client] rgw s3 auth use sts = True 2026-03-20T11:22:01.711 INFO:tasks.ceph:[client] rgw sts key = abcdefghijklmnoq 2026-03-20T11:22:01.711 INFO:tasks.ceph:[client] rgw sync data inject err probability = 0 2026-03-20T11:22:01.711 INFO:tasks.ceph:[client] rgw sync log trim interval = 0 2026-03-20T11:22:01.711 INFO:tasks.ceph:[client] rgw sync meta inject err probability = 0 2026-03-20T11:22:01.711 INFO:tasks.ceph:[client] rgw sync obj etag verify = True 2026-03-20T11:22:01.711 INFO:tasks.ceph:[client] setgroup = ceph 2026-03-20T11:22:01.711 INFO:tasks.ceph:[client] setuser = ceph 2026-03-20T11:22:01.711 INFO:tasks.ceph:[client.0] rgw data notify interval msec = 0 2026-03-20T11:22:01.711 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-03-20T11:22:01.711 INFO:tasks.ceph:[mgr] debug ms = 1 2026-03-20T11:22:01.711 INFO:tasks.ceph:[mon] debug mon = 20 2026-03-20T11:22:01.711 INFO:tasks.ceph:[mon] debug ms = 1 2026-03-20T11:22:01.711 INFO:tasks.ceph:[mon] debug paxos = 20 2026-03-20T11:22:01.711 INFO:tasks.ceph:[osd] debug ms = 1 2026-03-20T11:22:01.711 INFO:tasks.ceph:[osd] debug osd = 20 2026-03-20T11:22:01.711 INFO:tasks.ceph:[osd] osd mclock iops capacity threshold hdd = 49000 2026-03-20T11:22:01.711 INFO:tasks.ceph:[osd] rocksdb delete range threshold = 0 2026-03-20T11:22:01.711 INFO:tasks.ceph:[osd.0] osd_max_omap_entries_per_request = 10 2026-03-20T11:22:01.711 INFO:tasks.ceph:[osd.1] osd_max_omap_entries_per_request = 1000 2026-03-20T11:22:01.711 INFO:tasks.ceph:[osd.2] osd_max_omap_entries_per_request = 10000 2026-03-20T11:22:01.711 INFO:tasks.ceph:Setting up c2.mon.a... 2026-03-20T11:22:01.711 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/c2.keyring 2026-03-20T11:22:01.750 INFO:teuthology.orchestra.run.vm07.stdout:creating /etc/ceph/c2.keyring 2026-03-20T11:22:01.752 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/c2.keyring 2026-03-20T11:22:01.831 DEBUG:teuthology.orchestra.run.vm07:> sudo chmod 0644 /etc/ceph/c2.keyring 2026-03-20T11:22:01.896 DEBUG:tasks.ceph:Ceph mon addresses: [('c2.mon.a', '192.168.123.107')] 2026-03-20T11:22:01.896 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '192.168.123.107'}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'debug ms': 1, 'debug osd': 20, 'osd mclock iops capacity threshold hdd': 49000, 'rocksdb delete range threshold': 0}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rgw': 20, 'debug rgw lifecycle': 20, 'debug rgw notification': 20, 'debug rgw sync': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=', 'rgw curl low speed time': 300, 'rgw data log num shards': 4, 'rgw data sync poll interval': 5, 'rgw md log max shards': 4, 'rgw meta sync poll interval': 5, 'rgw s3 auth use sts': True, 'rgw sts key': 'abcdefghijklmnoq', 'rgw sync data inject err probability': 0, 'rgw sync log trim interval': 0, 'rgw sync meta inject err probability': 0, 'rgw sync obj etag verify': True, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'mon.a': {}, 'client.0': {'rgw data notify interval msec': 0}, 'osd.0': {'osd_max_omap_entries_per_request': 10}, 'osd.1': {'osd_max_omap_entries_per_request': 1000}, 'osd.2': {'osd_max_omap_entries_per_request': 10000}} 2026-03-20T11:22:01.897 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-20T11:22:01.897 DEBUG:teuthology.orchestra.run.vm07:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-03-20T11:22:01.953 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 192.168.123.107 --print /home/ubuntu/cephtest/c2.monmap 2026-03-20T11:22:02.027 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:22:02.027 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:22:02.028 INFO:teuthology.orchestra.run.vm07.stdout:monmaptool: monmap file /home/ubuntu/cephtest/c2.monmap 2026-03-20T11:22:02.028 INFO:teuthology.orchestra.run.vm07.stdout:monmaptool: generated fsid 9e939ad0-eb90-4072-8220-014224b87802 2026-03-20T11:22:02.028 INFO:teuthology.orchestra.run.vm07.stdout:setting min_mon_release = tentacle 2026-03-20T11:22:02.028 INFO:teuthology.orchestra.run.vm07.stdout:epoch 0 2026-03-20T11:22:02.028 INFO:teuthology.orchestra.run.vm07.stdout:fsid 9e939ad0-eb90-4072-8220-014224b87802 2026-03-20T11:22:02.028 INFO:teuthology.orchestra.run.vm07.stdout:last_changed 2026-03-20T11:22:02.030559+0000 2026-03-20T11:22:02.028 INFO:teuthology.orchestra.run.vm07.stdout:created 2026-03-20T11:22:02.030559+0000 2026-03-20T11:22:02.028 INFO:teuthology.orchestra.run.vm07.stdout:min_mon_release 20 (tentacle) 2026-03-20T11:22:02.028 INFO:teuthology.orchestra.run.vm07.stdout:election_strategy: 1 2026-03-20T11:22:02.028 INFO:teuthology.orchestra.run.vm07.stdout:0: [v2:192.168.123.107:3300/0,v1:192.168.123.107:6789/0] mon.a 2026-03-20T11:22:02.028 INFO:teuthology.orchestra.run.vm07.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/c2.monmap (1 monitors) 2026-03-20T11:22:02.030 DEBUG:teuthology.orchestra.run.vm07:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-03-20T11:22:02.085 INFO:tasks.ceph:Writing /etc/ceph/c2.conf for FSID 9e939ad0-eb90-4072-8220-014224b87802... 2026-03-20T11:22:02.085 DEBUG:teuthology.orchestra.run.vm02:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/c2.conf && sudo chmod 0644 /etc/ceph/c2.conf > /dev/null 2026-03-20T11:22:02.088 DEBUG:teuthology.orchestra.run.vm05:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/c2.conf && sudo chmod 0644 /etc/ceph/c2.conf > /dev/null 2026-03-20T11:22:02.090 DEBUG:teuthology.orchestra.run.vm07:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/c2.conf && sudo chmod 0644 /etc/ceph/c2.conf > /dev/null 2026-03-20T11:22:02.127 DEBUG:teuthology.orchestra.run.vm08:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/c2.conf && sudo chmod 0644 /etc/ceph/c2.conf > /dev/null 2026-03-20T11:22:02.136 INFO:teuthology.orchestra.run.vm02.stdout:[global] 2026-03-20T11:22:02.136 INFO:teuthology.orchestra.run.vm02.stdout: chdir = "" 2026-03-20T11:22:02.136 INFO:teuthology.orchestra.run.vm02.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-03-20T11:22:02.136 INFO:teuthology.orchestra.run.vm02.stdout: auth supported = cephx 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: filestore xattr use omap = true 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: mon clock drift allowed = 1.000 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: osd crush chooseleaf type = 0 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: auth debug = true 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: ms die on old message = true 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: ms die on bug = true 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: mon max pg per osd = 10000 # >= luminous 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: mon pg warn max object skew = 0 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: # disable pg_autoscaler by default for new pools 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: osd_pool_default_pg_autoscale_mode = off 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: osd pool default size = 2 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: mon osd allow primary affinity = true 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: mon osd allow pg remap = true 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: mon warn on legacy crush tunables = false 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: mon warn on crush straw calc version zero = false 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: mon warn on no sortbitwise = false 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: mon warn on osd down out interval zero = false 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: mon warn on too few osds = false 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: mon_warn_on_pool_no_redundancy = false 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: mon_allow_pool_size_one = true 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: osd default data pool replay window = 5 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: mon allow pool delete = true 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: mon cluster log file level = debug 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: debug asserts on shutdown = true 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: mon health detail to clog = false 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: mon host = 192.168.123.107 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: fsid = 9e939ad0-eb90-4072-8220-014224b87802 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout:[osd] 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: osd journal size = 100 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: osd scrub load threshold = 5.0 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: osd scrub max interval = 600 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: osd mclock profile = high_recovery_ops 2026-03-20T11:22:02.137 INFO:teuthology.orchestra.run.vm02.stdout: osd mclock skip benchmark = true 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: osd recover clone overlap = true 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: osd recovery max chunk = 1048576 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: osd debug shutdown = true 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: osd debug op order = true 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: osd debug verify stray on activate = true 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: osd debug trim objects = true 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: osd open classes on start = true 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: osd debug pg log writeout = true 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: osd deep scrub update digest min age = 30 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: osd map max advance = 10 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: journal zero on create = true 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: filestore ondisk finisher threads = 3 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: filestore apply finisher threads = 3 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: bdev debug aio = true 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: osd debug misdirected ops = true 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: debug ms = 1 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: debug osd = 20 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: osd mclock iops capacity threshold hdd = 49000 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: rocksdb delete range threshold = 0 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout:[mgr] 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: debug ms = 1 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: debug mgr = 20 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: debug mon = 20 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: debug auth = 20 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: mon reweight min pgs per osd = 4 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: mon reweight min bytes per osd = 10 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: mgr/telemetry/nag = false 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout:[mon] 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: debug ms = 1 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: debug mon = 20 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: debug paxos = 20 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: debug auth = 20 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: mon data avail warn = 5 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: mon mgr mkfs grace = 240 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: mon reweight min pgs per osd = 4 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: mon osd reporter subtree level = osd 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: mon osd prime pg temp = true 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: mon reweight min bytes per osd = 10 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: auth mon ticket ttl = 660 # 11m 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: auth service ticket ttl = 240 # 4m 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: # don't complain about insecure global_id in the test suite 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: # 1m isn't quite enough 2026-03-20T11:22:02.138 INFO:teuthology.orchestra.run.vm02.stdout: mon_down_mkfs_grace = 2m 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: mon_warn_on_filestore_osds = false 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout:[client] 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: rgw cache enabled = true 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: rgw enable ops log = true 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: rgw enable usage log = true 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: debug rgw = 20 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: debug rgw lifecycle = 20 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: debug rgw notification = 20 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: debug rgw sync = 20 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: rgw crypt require ssl = False 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: rgw crypt s3 kms backend = testing 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: rgw curl low speed time = 300 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: rgw data log num shards = 4 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: rgw data sync poll interval = 5 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: rgw md log max shards = 4 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: rgw meta sync poll interval = 5 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: rgw s3 auth use sts = True 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: rgw sts key = abcdefghijklmnoq 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: rgw sync data inject err probability = 0 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: rgw sync log trim interval = 0 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: rgw sync meta inject err probability = 0 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: rgw sync obj etag verify = True 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: setgroup = ceph 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: setuser = ceph 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout:[mon.a] 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout:[client.0] 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: rgw data notify interval msec = 0 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout:[osd.0] 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: osd_max_omap_entries_per_request = 10 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout:[osd.1] 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: osd_max_omap_entries_per_request = 1000 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout:[osd.2] 2026-03-20T11:22:02.139 INFO:teuthology.orchestra.run.vm02.stdout: osd_max_omap_entries_per_request = 10000 2026-03-20T11:22:02.144 INFO:teuthology.orchestra.run.vm05.stdout:[global] 2026-03-20T11:22:02.144 INFO:teuthology.orchestra.run.vm05.stdout: chdir = "" 2026-03-20T11:22:02.144 INFO:teuthology.orchestra.run.vm05.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-03-20T11:22:02.144 INFO:teuthology.orchestra.run.vm05.stdout: auth supported = cephx 2026-03-20T11:22:02.144 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.144 INFO:teuthology.orchestra.run.vm05.stdout: filestore xattr use omap = true 2026-03-20T11:22:02.144 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.144 INFO:teuthology.orchestra.run.vm05.stdout: mon clock drift allowed = 1.000 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: osd crush chooseleaf type = 0 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: auth debug = true 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: ms die on old message = true 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: ms die on bug = true 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: mon max pg per osd = 10000 # >= luminous 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: mon pg warn max object skew = 0 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: # disable pg_autoscaler by default for new pools 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: osd_pool_default_pg_autoscale_mode = off 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: osd pool default size = 2 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: mon osd allow primary affinity = true 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: mon osd allow pg remap = true 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: mon warn on legacy crush tunables = false 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: mon warn on crush straw calc version zero = false 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: mon warn on no sortbitwise = false 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: mon warn on osd down out interval zero = false 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: mon warn on too few osds = false 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: mon_warn_on_pool_no_redundancy = false 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: mon_allow_pool_size_one = true 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: osd default data pool replay window = 5 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: mon allow pool delete = true 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: mon cluster log file level = debug 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: debug asserts on shutdown = true 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: mon health detail to clog = false 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: mon host = 192.168.123.107 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: fsid = 9e939ad0-eb90-4072-8220-014224b87802 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout:[osd] 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: osd journal size = 100 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: osd scrub load threshold = 5.0 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: osd scrub max interval = 600 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: osd mclock profile = high_recovery_ops 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: osd mclock skip benchmark = true 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: osd recover clone overlap = true 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: osd recovery max chunk = 1048576 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: osd debug shutdown = true 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: osd debug op order = true 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: osd debug verify stray on activate = true 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: osd debug trim objects = true 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: osd open classes on start = true 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: osd debug pg log writeout = true 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: osd deep scrub update digest min age = 30 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: osd map max advance = 10 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.145 INFO:teuthology.orchestra.run.vm05.stdout: journal zero on create = true 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: filestore ondisk finisher threads = 3 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: filestore apply finisher threads = 3 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: bdev debug aio = true 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: osd debug misdirected ops = true 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: debug ms = 1 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: debug osd = 20 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: osd mclock iops capacity threshold hdd = 49000 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: rocksdb delete range threshold = 0 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout:[mgr] 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: debug ms = 1 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: debug mgr = 20 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: debug mon = 20 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: debug auth = 20 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: mon reweight min pgs per osd = 4 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: mon reweight min bytes per osd = 10 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: mgr/telemetry/nag = false 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout:[mon] 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: debug ms = 1 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: debug mon = 20 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: debug paxos = 20 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: debug auth = 20 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: mon data avail warn = 5 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: mon mgr mkfs grace = 240 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: mon reweight min pgs per osd = 4 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: mon osd reporter subtree level = osd 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: mon osd prime pg temp = true 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: mon reweight min bytes per osd = 10 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: auth mon ticket ttl = 660 # 11m 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: auth service ticket ttl = 240 # 4m 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: # don't complain about insecure global_id in the test suite 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: # 1m isn't quite enough 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: mon_down_mkfs_grace = 2m 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: mon_warn_on_filestore_osds = false 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout:[client] 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: rgw cache enabled = true 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: rgw enable ops log = true 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: rgw enable usage log = true 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: debug rgw = 20 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: debug rgw lifecycle = 20 2026-03-20T11:22:02.146 INFO:teuthology.orchestra.run.vm05.stdout: debug rgw notification = 20 2026-03-20T11:22:02.147 INFO:teuthology.orchestra.run.vm05.stdout: debug rgw sync = 20 2026-03-20T11:22:02.147 INFO:teuthology.orchestra.run.vm05.stdout: rgw crypt require ssl = False 2026-03-20T11:22:02.147 INFO:teuthology.orchestra.run.vm05.stdout: rgw crypt s3 kms backend = testing 2026-03-20T11:22:02.147 INFO:teuthology.orchestra.run.vm05.stdout: rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= 2026-03-20T11:22:02.147 INFO:teuthology.orchestra.run.vm05.stdout: rgw curl low speed time = 300 2026-03-20T11:22:02.147 INFO:teuthology.orchestra.run.vm05.stdout: rgw data log num shards = 4 2026-03-20T11:22:02.147 INFO:teuthology.orchestra.run.vm05.stdout: rgw data sync poll interval = 5 2026-03-20T11:22:02.147 INFO:teuthology.orchestra.run.vm05.stdout: rgw md log max shards = 4 2026-03-20T11:22:02.147 INFO:teuthology.orchestra.run.vm05.stdout: rgw meta sync poll interval = 5 2026-03-20T11:22:02.147 INFO:teuthology.orchestra.run.vm05.stdout: rgw s3 auth use sts = True 2026-03-20T11:22:02.147 INFO:teuthology.orchestra.run.vm05.stdout: rgw sts key = abcdefghijklmnoq 2026-03-20T11:22:02.147 INFO:teuthology.orchestra.run.vm05.stdout: rgw sync data inject err probability = 0 2026-03-20T11:22:02.147 INFO:teuthology.orchestra.run.vm05.stdout: rgw sync log trim interval = 0 2026-03-20T11:22:02.147 INFO:teuthology.orchestra.run.vm05.stdout: rgw sync meta inject err probability = 0 2026-03-20T11:22:02.147 INFO:teuthology.orchestra.run.vm05.stdout: rgw sync obj etag verify = True 2026-03-20T11:22:02.147 INFO:teuthology.orchestra.run.vm05.stdout: setgroup = ceph 2026-03-20T11:22:02.147 INFO:teuthology.orchestra.run.vm05.stdout: setuser = ceph 2026-03-20T11:22:02.147 INFO:teuthology.orchestra.run.vm05.stdout:[mon.a] 2026-03-20T11:22:02.147 INFO:teuthology.orchestra.run.vm05.stdout:[client.0] 2026-03-20T11:22:02.147 INFO:teuthology.orchestra.run.vm05.stdout: rgw data notify interval msec = 0 2026-03-20T11:22:02.147 INFO:teuthology.orchestra.run.vm05.stdout:[osd.0] 2026-03-20T11:22:02.147 INFO:teuthology.orchestra.run.vm05.stdout: osd_max_omap_entries_per_request = 10 2026-03-20T11:22:02.147 INFO:teuthology.orchestra.run.vm05.stdout:[osd.1] 2026-03-20T11:22:02.147 INFO:teuthology.orchestra.run.vm05.stdout: osd_max_omap_entries_per_request = 1000 2026-03-20T11:22:02.147 INFO:teuthology.orchestra.run.vm05.stdout:[osd.2] 2026-03-20T11:22:02.147 INFO:teuthology.orchestra.run.vm05.stdout: osd_max_omap_entries_per_request = 10000 2026-03-20T11:22:02.168 INFO:teuthology.orchestra.run.vm07.stdout:[global] 2026-03-20T11:22:02.168 INFO:teuthology.orchestra.run.vm07.stdout: chdir = "" 2026-03-20T11:22:02.168 INFO:teuthology.orchestra.run.vm07.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-03-20T11:22:02.168 INFO:teuthology.orchestra.run.vm07.stdout: auth supported = cephx 2026-03-20T11:22:02.168 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.168 INFO:teuthology.orchestra.run.vm07.stdout: filestore xattr use omap = true 2026-03-20T11:22:02.168 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.168 INFO:teuthology.orchestra.run.vm07.stdout: mon clock drift allowed = 1.000 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: osd crush chooseleaf type = 0 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: auth debug = true 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: ms die on old message = true 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: ms die on bug = true 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: mon max pg per osd = 10000 # >= luminous 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: mon pg warn max object skew = 0 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: # disable pg_autoscaler by default for new pools 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: osd_pool_default_pg_autoscale_mode = off 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: osd pool default size = 2 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: mon osd allow primary affinity = true 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: mon osd allow pg remap = true 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: mon warn on legacy crush tunables = false 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: mon warn on crush straw calc version zero = false 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: mon warn on no sortbitwise = false 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: mon warn on osd down out interval zero = false 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: mon warn on too few osds = false 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: mon_warn_on_pool_no_redundancy = false 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: mon_allow_pool_size_one = true 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: osd default data pool replay window = 5 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: mon allow pool delete = true 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: mon cluster log file level = debug 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: debug asserts on shutdown = true 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: mon health detail to clog = false 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: mon host = 192.168.123.107 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: fsid = 9e939ad0-eb90-4072-8220-014224b87802 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout:[osd] 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: osd journal size = 100 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: osd scrub load threshold = 5.0 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: osd scrub max interval = 600 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: osd mclock profile = high_recovery_ops 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: osd mclock skip benchmark = true 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: osd recover clone overlap = true 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: osd recovery max chunk = 1048576 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: osd debug shutdown = true 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: osd debug op order = true 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: osd debug verify stray on activate = true 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: osd debug trim objects = true 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: osd open classes on start = true 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: osd debug pg log writeout = true 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: osd deep scrub update digest min age = 30 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.169 INFO:teuthology.orchestra.run.vm07.stdout: osd map max advance = 10 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: journal zero on create = true 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: filestore ondisk finisher threads = 3 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: filestore apply finisher threads = 3 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: bdev debug aio = true 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: osd debug misdirected ops = true 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: debug ms = 1 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: debug osd = 20 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: osd mclock iops capacity threshold hdd = 49000 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: rocksdb delete range threshold = 0 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout:[mgr] 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: debug ms = 1 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: debug mgr = 20 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: debug mon = 20 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: debug auth = 20 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: mon reweight min pgs per osd = 4 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: mon reweight min bytes per osd = 10 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: mgr/telemetry/nag = false 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout:[mon] 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: debug ms = 1 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: debug mon = 20 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: debug paxos = 20 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: debug auth = 20 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: mon data avail warn = 5 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: mon mgr mkfs grace = 240 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: mon reweight min pgs per osd = 4 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: mon osd reporter subtree level = osd 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: mon osd prime pg temp = true 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: mon reweight min bytes per osd = 10 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: auth mon ticket ttl = 660 # 11m 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: auth service ticket ttl = 240 # 4m 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: # don't complain about insecure global_id in the test suite 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: # 1m isn't quite enough 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: mon_down_mkfs_grace = 2m 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: mon_warn_on_filestore_osds = false 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout:[client] 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: rgw cache enabled = true 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: rgw enable ops log = true 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: rgw enable usage log = true 2026-03-20T11:22:02.170 INFO:teuthology.orchestra.run.vm07.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout: debug rgw = 20 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout: debug rgw lifecycle = 20 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout: debug rgw notification = 20 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout: debug rgw sync = 20 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout: rgw crypt require ssl = False 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout: rgw crypt s3 kms backend = testing 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout: rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout: rgw curl low speed time = 300 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout: rgw data log num shards = 4 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout: rgw data sync poll interval = 5 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout: rgw md log max shards = 4 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout: rgw meta sync poll interval = 5 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout: rgw s3 auth use sts = True 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout: rgw sts key = abcdefghijklmnoq 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout: rgw sync data inject err probability = 0 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout: rgw sync log trim interval = 0 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout: rgw sync meta inject err probability = 0 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout: rgw sync obj etag verify = True 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout: setgroup = ceph 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout: setuser = ceph 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout:[mon.a] 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout:[client.0] 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout: rgw data notify interval msec = 0 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout:[osd.0] 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout: osd_max_omap_entries_per_request = 10 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout:[osd.1] 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout: osd_max_omap_entries_per_request = 1000 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout:[osd.2] 2026-03-20T11:22:02.171 INFO:teuthology.orchestra.run.vm07.stdout: osd_max_omap_entries_per_request = 10000 2026-03-20T11:22:02.173 INFO:teuthology.orchestra.run.vm08.stdout:[global] 2026-03-20T11:22:02.173 INFO:teuthology.orchestra.run.vm08.stdout: chdir = "" 2026-03-20T11:22:02.173 INFO:teuthology.orchestra.run.vm08.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-03-20T11:22:02.173 INFO:teuthology.orchestra.run.vm08.stdout: auth supported = cephx 2026-03-20T11:22:02.173 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.173 INFO:teuthology.orchestra.run.vm08.stdout: filestore xattr use omap = true 2026-03-20T11:22:02.173 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.173 INFO:teuthology.orchestra.run.vm08.stdout: mon clock drift allowed = 1.000 2026-03-20T11:22:02.173 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.173 INFO:teuthology.orchestra.run.vm08.stdout: osd crush chooseleaf type = 0 2026-03-20T11:22:02.173 INFO:teuthology.orchestra.run.vm08.stdout: auth debug = true 2026-03-20T11:22:02.173 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.173 INFO:teuthology.orchestra.run.vm08.stdout: ms die on old message = true 2026-03-20T11:22:02.173 INFO:teuthology.orchestra.run.vm08.stdout: ms die on bug = true 2026-03-20T11:22:02.173 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.173 INFO:teuthology.orchestra.run.vm08.stdout: mon max pg per osd = 10000 # >= luminous 2026-03-20T11:22:02.173 INFO:teuthology.orchestra.run.vm08.stdout: mon pg warn max object skew = 0 2026-03-20T11:22:02.173 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.173 INFO:teuthology.orchestra.run.vm08.stdout: # disable pg_autoscaler by default for new pools 2026-03-20T11:22:02.173 INFO:teuthology.orchestra.run.vm08.stdout: osd_pool_default_pg_autoscale_mode = off 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: osd pool default size = 2 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: mon osd allow primary affinity = true 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: mon osd allow pg remap = true 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: mon warn on legacy crush tunables = false 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: mon warn on crush straw calc version zero = false 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: mon warn on no sortbitwise = false 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: mon warn on osd down out interval zero = false 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: mon warn on too few osds = false 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: mon_warn_on_pool_no_redundancy = false 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: mon_allow_pool_size_one = true 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: osd default data pool replay window = 5 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: mon allow pool delete = true 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: mon cluster log file level = debug 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: debug asserts on shutdown = true 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: mon health detail to clog = false 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: mon host = 192.168.123.107 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: fsid = 9e939ad0-eb90-4072-8220-014224b87802 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout:[osd] 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: osd journal size = 100 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: osd scrub load threshold = 5.0 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: osd scrub max interval = 600 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: osd mclock profile = high_recovery_ops 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: osd mclock skip benchmark = true 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: osd recover clone overlap = true 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: osd recovery max chunk = 1048576 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: osd debug shutdown = true 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: osd debug op order = true 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: osd debug verify stray on activate = true 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: osd debug trim objects = true 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: osd open classes on start = true 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: osd debug pg log writeout = true 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: osd deep scrub update digest min age = 30 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: osd map max advance = 10 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: journal zero on create = true 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: filestore ondisk finisher threads = 3 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: filestore apply finisher threads = 3 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: bdev debug aio = true 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: osd debug misdirected ops = true 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: debug ms = 1 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: debug osd = 20 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: osd mclock iops capacity threshold hdd = 49000 2026-03-20T11:22:02.174 INFO:teuthology.orchestra.run.vm08.stdout: rocksdb delete range threshold = 0 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout:[mgr] 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: debug ms = 1 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: debug mgr = 20 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: debug mon = 20 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: debug auth = 20 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: mon reweight min pgs per osd = 4 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: mon reweight min bytes per osd = 10 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: mgr/telemetry/nag = false 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout:[mon] 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: debug ms = 1 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: debug mon = 20 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: debug paxos = 20 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: debug auth = 20 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: mon data avail warn = 5 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: mon mgr mkfs grace = 240 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: mon reweight min pgs per osd = 4 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: mon osd reporter subtree level = osd 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: mon osd prime pg temp = true 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: mon reweight min bytes per osd = 10 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: auth mon ticket ttl = 660 # 11m 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: auth service ticket ttl = 240 # 4m 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: # don't complain about insecure global_id in the test suite 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: # 1m isn't quite enough 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: mon_down_mkfs_grace = 2m 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: mon_warn_on_filestore_osds = false 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout:[client] 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: rgw cache enabled = true 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: rgw enable ops log = true 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: rgw enable usage log = true 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: debug rgw = 20 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: debug rgw lifecycle = 20 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: debug rgw notification = 20 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: debug rgw sync = 20 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: rgw crypt require ssl = False 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: rgw crypt s3 kms backend = testing 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: rgw curl low speed time = 300 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: rgw data log num shards = 4 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: rgw data sync poll interval = 5 2026-03-20T11:22:02.175 INFO:teuthology.orchestra.run.vm08.stdout: rgw md log max shards = 4 2026-03-20T11:22:02.176 INFO:teuthology.orchestra.run.vm08.stdout: rgw meta sync poll interval = 5 2026-03-20T11:22:02.176 INFO:teuthology.orchestra.run.vm08.stdout: rgw s3 auth use sts = True 2026-03-20T11:22:02.176 INFO:teuthology.orchestra.run.vm08.stdout: rgw sts key = abcdefghijklmnoq 2026-03-20T11:22:02.176 INFO:teuthology.orchestra.run.vm08.stdout: rgw sync data inject err probability = 0 2026-03-20T11:22:02.176 INFO:teuthology.orchestra.run.vm08.stdout: rgw sync log trim interval = 0 2026-03-20T11:22:02.176 INFO:teuthology.orchestra.run.vm08.stdout: rgw sync meta inject err probability = 0 2026-03-20T11:22:02.176 INFO:teuthology.orchestra.run.vm08.stdout: rgw sync obj etag verify = True 2026-03-20T11:22:02.176 INFO:teuthology.orchestra.run.vm08.stdout: setgroup = ceph 2026-03-20T11:22:02.176 INFO:teuthology.orchestra.run.vm08.stdout: setuser = ceph 2026-03-20T11:22:02.176 INFO:teuthology.orchestra.run.vm08.stdout:[mon.a] 2026-03-20T11:22:02.176 INFO:teuthology.orchestra.run.vm08.stdout:[client.0] 2026-03-20T11:22:02.176 INFO:teuthology.orchestra.run.vm08.stdout: rgw data notify interval msec = 0 2026-03-20T11:22:02.176 INFO:teuthology.orchestra.run.vm08.stdout:[osd.0] 2026-03-20T11:22:02.176 INFO:teuthology.orchestra.run.vm08.stdout: osd_max_omap_entries_per_request = 10 2026-03-20T11:22:02.176 INFO:teuthology.orchestra.run.vm08.stdout:[osd.1] 2026-03-20T11:22:02.176 INFO:teuthology.orchestra.run.vm08.stdout: osd_max_omap_entries_per_request = 1000 2026-03-20T11:22:02.176 INFO:teuthology.orchestra.run.vm08.stdout:[osd.2] 2026-03-20T11:22:02.176 INFO:teuthology.orchestra.run.vm08.stdout: osd_max_omap_entries_per_request = 10000 2026-03-20T11:22:02.183 INFO:tasks.ceph:Creating admin key on c2.mon.a... 2026-03-20T11:22:02.183 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/c2.keyring 2026-03-20T11:22:02.277 INFO:tasks.ceph:Copying monmap to all nodes... 2026-03-20T11:22:02.277 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-20T11:22:02.277 DEBUG:teuthology.orchestra.run.vm07:> dd if=/etc/ceph/c2.keyring of=/dev/stdout 2026-03-20T11:22:02.296 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-20T11:22:02.296 DEBUG:teuthology.orchestra.run.vm07:> dd if=/home/ubuntu/cephtest/c2.monmap of=/dev/stdout 2026-03-20T11:22:02.355 INFO:tasks.ceph:Sending monmap to node ubuntu@vm02.local 2026-03-20T11:22:02.355 DEBUG:teuthology.orchestra.run.vm02:> set -ex 2026-03-20T11:22:02.355 DEBUG:teuthology.orchestra.run.vm02:> sudo dd of=/etc/ceph/c2.keyring 2026-03-20T11:22:02.355 DEBUG:teuthology.orchestra.run.vm02:> sudo chmod 0644 /etc/ceph/c2.keyring 2026-03-20T11:22:02.388 DEBUG:teuthology.orchestra.run.vm02:> set -ex 2026-03-20T11:22:02.388 DEBUG:teuthology.orchestra.run.vm02:> dd of=/home/ubuntu/cephtest/c2.monmap 2026-03-20T11:22:02.442 INFO:tasks.ceph:Sending monmap to node ubuntu@vm05.local 2026-03-20T11:22:02.442 DEBUG:teuthology.orchestra.run.vm05:> set -ex 2026-03-20T11:22:02.442 DEBUG:teuthology.orchestra.run.vm05:> sudo dd of=/etc/ceph/c2.keyring 2026-03-20T11:22:02.442 DEBUG:teuthology.orchestra.run.vm05:> sudo chmod 0644 /etc/ceph/c2.keyring 2026-03-20T11:22:02.484 DEBUG:teuthology.orchestra.run.vm05:> set -ex 2026-03-20T11:22:02.484 DEBUG:teuthology.orchestra.run.vm05:> dd of=/home/ubuntu/cephtest/c2.monmap 2026-03-20T11:22:02.542 INFO:tasks.ceph:Sending monmap to node ubuntu@vm07.local 2026-03-20T11:22:02.542 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-20T11:22:02.542 DEBUG:teuthology.orchestra.run.vm07:> sudo dd of=/etc/ceph/c2.keyring 2026-03-20T11:22:02.542 DEBUG:teuthology.orchestra.run.vm07:> sudo chmod 0644 /etc/ceph/c2.keyring 2026-03-20T11:22:02.582 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-20T11:22:02.582 DEBUG:teuthology.orchestra.run.vm07:> dd of=/home/ubuntu/cephtest/c2.monmap 2026-03-20T11:22:02.642 INFO:tasks.ceph:Sending monmap to node ubuntu@vm08.local 2026-03-20T11:22:02.642 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-20T11:22:02.642 DEBUG:teuthology.orchestra.run.vm08:> sudo dd of=/etc/ceph/c2.keyring 2026-03-20T11:22:02.642 DEBUG:teuthology.orchestra.run.vm08:> sudo chmod 0644 /etc/ceph/c2.keyring 2026-03-20T11:22:02.679 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-20T11:22:02.679 DEBUG:teuthology.orchestra.run.vm08:> dd of=/home/ubuntu/cephtest/c2.monmap 2026-03-20T11:22:02.737 INFO:tasks.ceph:Setting up mon nodes... 2026-03-20T11:22:02.737 INFO:tasks.ceph:Setting up mgr nodes... 2026-03-20T11:22:02.737 DEBUG:teuthology.orchestra.run.vm08:> sudo mkdir -p /var/lib/ceph/mgr/c2-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/c2-x/keyring 2026-03-20T11:22:02.828 INFO:teuthology.orchestra.run.vm08.stdout:creating /var/lib/ceph/mgr/c2-x/keyring 2026-03-20T11:22:02.830 INFO:tasks.ceph:Setting up mds nodes... 2026-03-20T11:22:02.831 INFO:tasks.ceph_client:Setting up client nodes... 2026-03-20T11:22:02.831 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/c2.client.0.keyring && sudo chmod 0644 /etc/ceph/c2.client.0.keyring 2026-03-20T11:22:02.870 INFO:teuthology.orchestra.run.vm07.stdout:creating /etc/ceph/c2.client.0.keyring 2026-03-20T11:22:02.884 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.1 /etc/ceph/c2.client.1.keyring && sudo chmod 0644 /etc/ceph/c2.client.1.keyring 2026-03-20T11:22:02.922 INFO:teuthology.orchestra.run.vm08.stdout:creating /etc/ceph/c2.client.1.keyring 2026-03-20T11:22:02.936 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-03-20T11:22:02.936 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@vm02.local'): {'c1.osd.0': '/dev/vg_nvme/lv_1', 'c1.osd.1': '/dev/vg_nvme/lv_2', 'c1.osd.2': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@vm05.local'): {'c1.osd.3': '/dev/vg_nvme/lv_1', 'c1.osd.4': '/dev/vg_nvme/lv_2', 'c1.osd.5': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@vm07.local'): {'c2.osd.0': '/dev/vg_nvme/lv_1', 'c2.osd.1': '/dev/vg_nvme/lv_2', 'c2.osd.2': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@vm08.local'): {'c2.osd.3': '/dev/vg_nvme/lv_1', 'c2.osd.4': '/dev/vg_nvme/lv_2', 'c2.osd.5': '/dev/vg_nvme/lv_3'}} 2026-03-20T11:22:02.936 DEBUG:teuthology.orchestra.run.vm07:> sudo mkdir -p /var/lib/ceph/osd/c2-0 2026-03-20T11:22:02.965 INFO:tasks.ceph:roles_to_devs: {'c2.osd.0': '/dev/vg_nvme/lv_1', 'c2.osd.1': '/dev/vg_nvme/lv_2', 'c2.osd.2': '/dev/vg_nvme/lv_3'} 2026-03-20T11:22:02.965 INFO:tasks.ceph:role: c2.osd.0 2026-03-20T11:22:02.965 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@vm07.local 2026-03-20T11:22:02.965 DEBUG:teuthology.orchestra.run.vm07:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-03-20T11:22:03.034 INFO:teuthology.orchestra.run.vm07.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=1310464 blks 2026-03-20T11:22:03.035 INFO:teuthology.orchestra.run.vm07.stdout: = sectsz=512 attr=2, projid32bit=1 2026-03-20T11:22:03.035 INFO:teuthology.orchestra.run.vm07.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-03-20T11:22:03.035 INFO:teuthology.orchestra.run.vm07.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-03-20T11:22:03.035 INFO:teuthology.orchestra.run.vm07.stdout:data = bsize=4096 blocks=5241856, imaxpct=25 2026-03-20T11:22:03.035 INFO:teuthology.orchestra.run.vm07.stdout: = sunit=0 swidth=0 blks 2026-03-20T11:22:03.035 INFO:teuthology.orchestra.run.vm07.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-03-20T11:22:03.035 INFO:teuthology.orchestra.run.vm07.stdout:log =internal log bsize=4096 blocks=16384, version=2 2026-03-20T11:22:03.035 INFO:teuthology.orchestra.run.vm07.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2026-03-20T11:22:03.035 INFO:teuthology.orchestra.run.vm07.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-03-20T11:22:03.040 INFO:teuthology.orchestra.run.vm07.stdout:Discarding blocks...Done. 2026-03-20T11:22:03.042 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@vm07.local -o noatime 2026-03-20T11:22:03.042 DEBUG:teuthology.orchestra.run.vm07:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/c2-0 2026-03-20T11:22:03.115 DEBUG:teuthology.orchestra.run.vm07:> sudo /sbin/restorecon /var/lib/ceph/osd/c2-0 2026-03-20T11:22:03.185 DEBUG:teuthology.orchestra.run.vm07:> sudo mkdir -p /var/lib/ceph/osd/c2-1 2026-03-20T11:22:03.251 INFO:tasks.ceph:roles_to_devs: {'c2.osd.0': '/dev/vg_nvme/lv_1', 'c2.osd.1': '/dev/vg_nvme/lv_2', 'c2.osd.2': '/dev/vg_nvme/lv_3'} 2026-03-20T11:22:03.251 INFO:tasks.ceph:role: c2.osd.1 2026-03-20T11:22:03.251 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@vm07.local 2026-03-20T11:22:03.251 DEBUG:teuthology.orchestra.run.vm07:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-03-20T11:22:03.317 INFO:teuthology.orchestra.run.vm07.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=1310464 blks 2026-03-20T11:22:03.317 INFO:teuthology.orchestra.run.vm07.stdout: = sectsz=512 attr=2, projid32bit=1 2026-03-20T11:22:03.317 INFO:teuthology.orchestra.run.vm07.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-03-20T11:22:03.317 INFO:teuthology.orchestra.run.vm07.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-03-20T11:22:03.317 INFO:teuthology.orchestra.run.vm07.stdout:data = bsize=4096 blocks=5241856, imaxpct=25 2026-03-20T11:22:03.317 INFO:teuthology.orchestra.run.vm07.stdout: = sunit=0 swidth=0 blks 2026-03-20T11:22:03.317 INFO:teuthology.orchestra.run.vm07.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-03-20T11:22:03.317 INFO:teuthology.orchestra.run.vm07.stdout:log =internal log bsize=4096 blocks=16384, version=2 2026-03-20T11:22:03.317 INFO:teuthology.orchestra.run.vm07.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2026-03-20T11:22:03.317 INFO:teuthology.orchestra.run.vm07.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-03-20T11:22:03.321 INFO:teuthology.orchestra.run.vm07.stdout:Discarding blocks...Done. 2026-03-20T11:22:03.322 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@vm07.local -o noatime 2026-03-20T11:22:03.322 DEBUG:teuthology.orchestra.run.vm07:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/c2-1 2026-03-20T11:22:03.390 DEBUG:teuthology.orchestra.run.vm07:> sudo /sbin/restorecon /var/lib/ceph/osd/c2-1 2026-03-20T11:22:03.457 DEBUG:teuthology.orchestra.run.vm07:> sudo mkdir -p /var/lib/ceph/osd/c2-2 2026-03-20T11:22:03.522 INFO:tasks.ceph:roles_to_devs: {'c2.osd.0': '/dev/vg_nvme/lv_1', 'c2.osd.1': '/dev/vg_nvme/lv_2', 'c2.osd.2': '/dev/vg_nvme/lv_3'} 2026-03-20T11:22:03.522 INFO:tasks.ceph:role: c2.osd.2 2026-03-20T11:22:03.522 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@vm07.local 2026-03-20T11:22:03.522 DEBUG:teuthology.orchestra.run.vm07:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-03-20T11:22:03.587 INFO:teuthology.orchestra.run.vm07.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=1310464 blks 2026-03-20T11:22:03.587 INFO:teuthology.orchestra.run.vm07.stdout: = sectsz=512 attr=2, projid32bit=1 2026-03-20T11:22:03.587 INFO:teuthology.orchestra.run.vm07.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-03-20T11:22:03.587 INFO:teuthology.orchestra.run.vm07.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-03-20T11:22:03.588 INFO:teuthology.orchestra.run.vm07.stdout:data = bsize=4096 blocks=5241856, imaxpct=25 2026-03-20T11:22:03.588 INFO:teuthology.orchestra.run.vm07.stdout: = sunit=0 swidth=0 blks 2026-03-20T11:22:03.588 INFO:teuthology.orchestra.run.vm07.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-03-20T11:22:03.588 INFO:teuthology.orchestra.run.vm07.stdout:log =internal log bsize=4096 blocks=16384, version=2 2026-03-20T11:22:03.588 INFO:teuthology.orchestra.run.vm07.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2026-03-20T11:22:03.588 INFO:teuthology.orchestra.run.vm07.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-03-20T11:22:03.592 INFO:teuthology.orchestra.run.vm07.stdout:Discarding blocks...Done. 2026-03-20T11:22:03.593 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@vm07.local -o noatime 2026-03-20T11:22:03.593 DEBUG:teuthology.orchestra.run.vm07:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/c2-2 2026-03-20T11:22:03.665 DEBUG:teuthology.orchestra.run.vm07:> sudo /sbin/restorecon /var/lib/ceph/osd/c2-2 2026-03-20T11:22:03.734 DEBUG:teuthology.orchestra.run.vm07:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster c2 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/c2.monmap 2026-03-20T11:22:03.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:03.818+0000 7fa85bc48900 -1 auth: error reading file: /var/lib/ceph/osd/c2-0/keyring: can't open /var/lib/ceph/osd/c2-0/keyring: (2) No such file or directory 2026-03-20T11:22:03.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:03.819+0000 7fa85bc48900 -1 created new key in keyring /var/lib/ceph/osd/c2-0/keyring 2026-03-20T11:22:03.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:03.819+0000 7fa85bc48900 -1 bdev(0x5581e3eb5800 /var/lib/ceph/osd/c2-0/block) open stat got: (1) Operation not permitted 2026-03-20T11:22:03.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:03.819+0000 7fa85bc48900 -1 bluestore(/var/lib/ceph/osd/c2-0) _read_fsid unparsable uuid 2026-03-20T11:22:04.301 DEBUG:teuthology.orchestra.run.vm07:> sudo chown -R ceph:ceph /var/lib/ceph/osd/c2-0 2026-03-20T11:22:04.327 DEBUG:teuthology.orchestra.run.vm07:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster c2 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/c2.monmap 2026-03-20T11:22:04.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:04.409+0000 7fb5ba085900 -1 auth: error reading file: /var/lib/ceph/osd/c2-1/keyring: can't open /var/lib/ceph/osd/c2-1/keyring: (2) No such file or directory 2026-03-20T11:22:04.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:04.409+0000 7fb5ba085900 -1 created new key in keyring /var/lib/ceph/osd/c2-1/keyring 2026-03-20T11:22:04.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:04.409+0000 7fb5ba085900 -1 bdev(0x56506c2d7800 /var/lib/ceph/osd/c2-1/block) open stat got: (1) Operation not permitted 2026-03-20T11:22:04.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:04.409+0000 7fb5ba085900 -1 bluestore(/var/lib/ceph/osd/c2-1) _read_fsid unparsable uuid 2026-03-20T11:22:06.283 DEBUG:teuthology.orchestra.run.vm07:> sudo chown -R ceph:ceph /var/lib/ceph/osd/c2-1 2026-03-20T11:22:06.350 DEBUG:teuthology.orchestra.run.vm07:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster c2 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/c2.monmap 2026-03-20T11:22:06.430 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:06.431+0000 7f880f210900 -1 auth: error reading file: /var/lib/ceph/osd/c2-2/keyring: can't open /var/lib/ceph/osd/c2-2/keyring: (2) No such file or directory 2026-03-20T11:22:06.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:06.431+0000 7f880f210900 -1 created new key in keyring /var/lib/ceph/osd/c2-2/keyring 2026-03-20T11:22:06.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:06.431+0000 7f880f210900 -1 bdev(0x5634c2e5d800 /var/lib/ceph/osd/c2-2/block) open stat got: (1) Operation not permitted 2026-03-20T11:22:06.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:06.432+0000 7f880f210900 -1 bluestore(/var/lib/ceph/osd/c2-2) _read_fsid unparsable uuid 2026-03-20T11:22:06.870 DEBUG:teuthology.orchestra.run.vm07:> sudo chown -R ceph:ceph /var/lib/ceph/osd/c2-2 2026-03-20T11:22:06.934 DEBUG:teuthology.orchestra.run.vm08:> sudo mkdir -p /var/lib/ceph/osd/c2-3 2026-03-20T11:22:06.957 INFO:tasks.ceph:roles_to_devs: {'c2.osd.3': '/dev/vg_nvme/lv_1', 'c2.osd.4': '/dev/vg_nvme/lv_2', 'c2.osd.5': '/dev/vg_nvme/lv_3'} 2026-03-20T11:22:06.957 INFO:tasks.ceph:role: c2.osd.3 2026-03-20T11:22:06.957 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@vm08.local 2026-03-20T11:22:06.957 DEBUG:teuthology.orchestra.run.vm08:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-03-20T11:22:07.021 INFO:teuthology.orchestra.run.vm08.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=1310464 blks 2026-03-20T11:22:07.021 INFO:teuthology.orchestra.run.vm08.stdout: = sectsz=512 attr=2, projid32bit=1 2026-03-20T11:22:07.021 INFO:teuthology.orchestra.run.vm08.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-03-20T11:22:07.021 INFO:teuthology.orchestra.run.vm08.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-03-20T11:22:07.021 INFO:teuthology.orchestra.run.vm08.stdout:data = bsize=4096 blocks=5241856, imaxpct=25 2026-03-20T11:22:07.021 INFO:teuthology.orchestra.run.vm08.stdout: = sunit=0 swidth=0 blks 2026-03-20T11:22:07.021 INFO:teuthology.orchestra.run.vm08.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-03-20T11:22:07.021 INFO:teuthology.orchestra.run.vm08.stdout:log =internal log bsize=4096 blocks=16384, version=2 2026-03-20T11:22:07.021 INFO:teuthology.orchestra.run.vm08.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2026-03-20T11:22:07.021 INFO:teuthology.orchestra.run.vm08.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-03-20T11:22:07.025 INFO:teuthology.orchestra.run.vm08.stdout:Discarding blocks...Done. 2026-03-20T11:22:07.027 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@vm08.local -o noatime 2026-03-20T11:22:07.027 DEBUG:teuthology.orchestra.run.vm08:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/c2-3 2026-03-20T11:22:07.096 DEBUG:teuthology.orchestra.run.vm08:> sudo /sbin/restorecon /var/lib/ceph/osd/c2-3 2026-03-20T11:22:07.160 DEBUG:teuthology.orchestra.run.vm08:> sudo mkdir -p /var/lib/ceph/osd/c2-4 2026-03-20T11:22:07.223 INFO:tasks.ceph:roles_to_devs: {'c2.osd.3': '/dev/vg_nvme/lv_1', 'c2.osd.4': '/dev/vg_nvme/lv_2', 'c2.osd.5': '/dev/vg_nvme/lv_3'} 2026-03-20T11:22:07.223 INFO:tasks.ceph:role: c2.osd.4 2026-03-20T11:22:07.223 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@vm08.local 2026-03-20T11:22:07.223 DEBUG:teuthology.orchestra.run.vm08:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-03-20T11:22:07.285 INFO:teuthology.orchestra.run.vm08.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=1310464 blks 2026-03-20T11:22:07.285 INFO:teuthology.orchestra.run.vm08.stdout: = sectsz=512 attr=2, projid32bit=1 2026-03-20T11:22:07.285 INFO:teuthology.orchestra.run.vm08.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-03-20T11:22:07.285 INFO:teuthology.orchestra.run.vm08.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-03-20T11:22:07.285 INFO:teuthology.orchestra.run.vm08.stdout:data = bsize=4096 blocks=5241856, imaxpct=25 2026-03-20T11:22:07.285 INFO:teuthology.orchestra.run.vm08.stdout: = sunit=0 swidth=0 blks 2026-03-20T11:22:07.285 INFO:teuthology.orchestra.run.vm08.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-03-20T11:22:07.285 INFO:teuthology.orchestra.run.vm08.stdout:log =internal log bsize=4096 blocks=16384, version=2 2026-03-20T11:22:07.285 INFO:teuthology.orchestra.run.vm08.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2026-03-20T11:22:07.285 INFO:teuthology.orchestra.run.vm08.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-03-20T11:22:07.289 INFO:teuthology.orchestra.run.vm08.stdout:Discarding blocks...Done. 2026-03-20T11:22:07.291 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@vm08.local -o noatime 2026-03-20T11:22:07.291 DEBUG:teuthology.orchestra.run.vm08:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/c2-4 2026-03-20T11:22:07.358 DEBUG:teuthology.orchestra.run.vm08:> sudo /sbin/restorecon /var/lib/ceph/osd/c2-4 2026-03-20T11:22:07.423 DEBUG:teuthology.orchestra.run.vm08:> sudo mkdir -p /var/lib/ceph/osd/c2-5 2026-03-20T11:22:07.484 INFO:tasks.ceph:roles_to_devs: {'c2.osd.3': '/dev/vg_nvme/lv_1', 'c2.osd.4': '/dev/vg_nvme/lv_2', 'c2.osd.5': '/dev/vg_nvme/lv_3'} 2026-03-20T11:22:07.484 INFO:tasks.ceph:role: c2.osd.5 2026-03-20T11:22:07.484 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@vm08.local 2026-03-20T11:22:07.484 DEBUG:teuthology.orchestra.run.vm08:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-03-20T11:22:07.546 INFO:teuthology.orchestra.run.vm08.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=1310464 blks 2026-03-20T11:22:07.546 INFO:teuthology.orchestra.run.vm08.stdout: = sectsz=512 attr=2, projid32bit=1 2026-03-20T11:22:07.546 INFO:teuthology.orchestra.run.vm08.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-03-20T11:22:07.546 INFO:teuthology.orchestra.run.vm08.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-03-20T11:22:07.546 INFO:teuthology.orchestra.run.vm08.stdout:data = bsize=4096 blocks=5241856, imaxpct=25 2026-03-20T11:22:07.546 INFO:teuthology.orchestra.run.vm08.stdout: = sunit=0 swidth=0 blks 2026-03-20T11:22:07.546 INFO:teuthology.orchestra.run.vm08.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-03-20T11:22:07.546 INFO:teuthology.orchestra.run.vm08.stdout:log =internal log bsize=4096 blocks=16384, version=2 2026-03-20T11:22:07.546 INFO:teuthology.orchestra.run.vm08.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2026-03-20T11:22:07.546 INFO:teuthology.orchestra.run.vm08.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-03-20T11:22:07.550 INFO:teuthology.orchestra.run.vm08.stdout:Discarding blocks...Done. 2026-03-20T11:22:07.551 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@vm08.local -o noatime 2026-03-20T11:22:07.551 DEBUG:teuthology.orchestra.run.vm08:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/c2-5 2026-03-20T11:22:07.619 DEBUG:teuthology.orchestra.run.vm08:> sudo /sbin/restorecon /var/lib/ceph/osd/c2-5 2026-03-20T11:22:07.684 DEBUG:teuthology.orchestra.run.vm08:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster c2 --mkfs --mkkey -i 3 --monmap /home/ubuntu/cephtest/c2.monmap 2026-03-20T11:22:07.761 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:07.762+0000 7fe3bc758900 -1 auth: error reading file: /var/lib/ceph/osd/c2-3/keyring: can't open /var/lib/ceph/osd/c2-3/keyring: (2) No such file or directory 2026-03-20T11:22:07.761 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:07.763+0000 7fe3bc758900 -1 created new key in keyring /var/lib/ceph/osd/c2-3/keyring 2026-03-20T11:22:07.761 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:07.763+0000 7fe3bc758900 -1 bdev(0x555e7b9f1800 /var/lib/ceph/osd/c2-3/block) open stat got: (1) Operation not permitted 2026-03-20T11:22:07.761 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:07.763+0000 7fe3bc758900 -1 bluestore(/var/lib/ceph/osd/c2-3) _read_fsid unparsable uuid 2026-03-20T11:22:08.214 DEBUG:teuthology.orchestra.run.vm08:> sudo chown -R ceph:ceph /var/lib/ceph/osd/c2-3 2026-03-20T11:22:08.278 DEBUG:teuthology.orchestra.run.vm08:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster c2 --mkfs --mkkey -i 4 --monmap /home/ubuntu/cephtest/c2.monmap 2026-03-20T11:22:08.353 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:08.355+0000 7f7a8a9b0900 -1 auth: error reading file: /var/lib/ceph/osd/c2-4/keyring: can't open /var/lib/ceph/osd/c2-4/keyring: (2) No such file or directory 2026-03-20T11:22:08.354 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:08.355+0000 7f7a8a9b0900 -1 created new key in keyring /var/lib/ceph/osd/c2-4/keyring 2026-03-20T11:22:08.354 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:08.355+0000 7f7a8a9b0900 -1 bdev(0x557cbcf47800 /var/lib/ceph/osd/c2-4/block) open stat got: (1) Operation not permitted 2026-03-20T11:22:08.354 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:08.356+0000 7f7a8a9b0900 -1 bluestore(/var/lib/ceph/osd/c2-4) _read_fsid unparsable uuid 2026-03-20T11:22:08.828 DEBUG:teuthology.orchestra.run.vm08:> sudo chown -R ceph:ceph /var/lib/ceph/osd/c2-4 2026-03-20T11:22:08.890 DEBUG:teuthology.orchestra.run.vm08:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster c2 --mkfs --mkkey -i 5 --monmap /home/ubuntu/cephtest/c2.monmap 2026-03-20T11:22:08.965 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:08.966+0000 7f5769012900 -1 auth: error reading file: /var/lib/ceph/osd/c2-5/keyring: can't open /var/lib/ceph/osd/c2-5/keyring: (2) No such file or directory 2026-03-20T11:22:08.965 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:08.967+0000 7f5769012900 -1 created new key in keyring /var/lib/ceph/osd/c2-5/keyring 2026-03-20T11:22:08.965 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:08.967+0000 7f5769012900 -1 bdev(0x55ce53d39800 /var/lib/ceph/osd/c2-5/block) open stat got: (1) Operation not permitted 2026-03-20T11:22:08.965 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:08.967+0000 7f5769012900 -1 bluestore(/var/lib/ceph/osd/c2-5) _read_fsid unparsable uuid 2026-03-20T11:22:09.389 DEBUG:teuthology.orchestra.run.vm08:> sudo chown -R ceph:ceph /var/lib/ceph/osd/c2-5 2026-03-20T11:22:09.458 INFO:tasks.ceph:Reading keys from all nodes... 2026-03-20T11:22:09.458 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-20T11:22:09.458 DEBUG:teuthology.orchestra.run.vm07:> sudo dd if=/var/lib/ceph/osd/c2-0/keyring of=/dev/stdout 2026-03-20T11:22:09.485 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-20T11:22:09.485 DEBUG:teuthology.orchestra.run.vm07:> sudo dd if=/var/lib/ceph/osd/c2-1/keyring of=/dev/stdout 2026-03-20T11:22:09.552 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-20T11:22:09.552 DEBUG:teuthology.orchestra.run.vm07:> sudo dd if=/var/lib/ceph/osd/c2-2/keyring of=/dev/stdout 2026-03-20T11:22:09.621 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-20T11:22:09.622 DEBUG:teuthology.orchestra.run.vm08:> sudo dd if=/var/lib/ceph/mgr/c2-x/keyring of=/dev/stdout 2026-03-20T11:22:09.650 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-20T11:22:09.651 DEBUG:teuthology.orchestra.run.vm08:> sudo dd if=/var/lib/ceph/osd/c2-3/keyring of=/dev/stdout 2026-03-20T11:22:09.717 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-20T11:22:09.718 DEBUG:teuthology.orchestra.run.vm08:> sudo dd if=/var/lib/ceph/osd/c2-4/keyring of=/dev/stdout 2026-03-20T11:22:09.784 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-20T11:22:09.784 DEBUG:teuthology.orchestra.run.vm08:> sudo dd if=/var/lib/ceph/osd/c2-5/keyring of=/dev/stdout 2026-03-20T11:22:09.853 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-20T11:22:09.853 DEBUG:teuthology.orchestra.run.vm07:> dd if=/etc/ceph/c2.client.0.keyring of=/dev/stdout 2026-03-20T11:22:09.869 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-20T11:22:09.869 DEBUG:teuthology.orchestra.run.vm08:> dd if=/etc/ceph/c2.client.1.keyring of=/dev/stdout 2026-03-20T11:22:09.912 INFO:tasks.ceph:Adding keys to all mons... 2026-03-20T11:22:09.912 DEBUG:teuthology.orchestra.run.vm07:> sudo tee -a /etc/ceph/c2.keyring 2026-03-20T11:22:09.938 INFO:teuthology.orchestra.run.vm07.stdout:[osd.0] 2026-03-20T11:22:09.938 INFO:teuthology.orchestra.run.vm07.stdout: key = AQDbLb1pTrbhMBAAsGJQu6wh8mgipjkVVhHk8Q== 2026-03-20T11:22:09.938 INFO:teuthology.orchestra.run.vm07.stdout:[osd.1] 2026-03-20T11:22:09.938 INFO:teuthology.orchestra.run.vm07.stdout: key = AQDcLb1pvB97GBAAYnUnnfUGkgxYgERvy8GROw== 2026-03-20T11:22:09.938 INFO:teuthology.orchestra.run.vm07.stdout:[osd.2] 2026-03-20T11:22:09.938 INFO:teuthology.orchestra.run.vm07.stdout: key = AQDeLb1p4STNGRAAv4qL1z92xp+s4dRf2Lgicg== 2026-03-20T11:22:09.938 INFO:teuthology.orchestra.run.vm07.stdout:[mgr.x] 2026-03-20T11:22:09.938 INFO:teuthology.orchestra.run.vm07.stdout: key = AQDaLb1pnSWCMRAA6Ygp355xfBWQqPx9nAGF2g== 2026-03-20T11:22:09.938 INFO:teuthology.orchestra.run.vm07.stdout:[osd.3] 2026-03-20T11:22:09.938 INFO:teuthology.orchestra.run.vm07.stdout: key = AQDfLb1pEK+ELRAArl13vCV6ato9c5pcRARCgQ== 2026-03-20T11:22:09.938 INFO:teuthology.orchestra.run.vm07.stdout:[osd.4] 2026-03-20T11:22:09.938 INFO:teuthology.orchestra.run.vm07.stdout: key = AQDgLb1pUKw/FRAA0tzciYiyMZZVbNlLrp4C8Q== 2026-03-20T11:22:09.938 INFO:teuthology.orchestra.run.vm07.stdout:[osd.5] 2026-03-20T11:22:09.938 INFO:teuthology.orchestra.run.vm07.stdout: key = AQDgLb1p7nysORAAP0yNxHoSUnHxEuI9+fRNmA== 2026-03-20T11:22:09.938 INFO:teuthology.orchestra.run.vm07.stdout:[client.0] 2026-03-20T11:22:09.938 INFO:teuthology.orchestra.run.vm07.stdout: key = AQDaLb1p6NX9MxAAsh/Gj6ASgdLijOEFNPk4Ag== 2026-03-20T11:22:09.938 INFO:teuthology.orchestra.run.vm07.stdout:[client.1] 2026-03-20T11:22:09.938 INFO:teuthology.orchestra.run.vm07.stdout: key = AQDaLb1puq4gNxAA1Uly0NukzINLlYXE5LsD8w== 2026-03-20T11:22:09.939 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c2.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-03-20T11:22:10.028 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c2.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-03-20T11:22:10.075 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c2.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-03-20T11:22:10.123 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c2.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-03-20T11:22:10.175 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c2.keyring --name=osd.3 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-03-20T11:22:10.224 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c2.keyring --name=osd.4 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-03-20T11:22:10.277 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c2.keyring --name=osd.5 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-03-20T11:22:10.333 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c2.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-20T11:22:10.389 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c2.keyring --name=client.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-20T11:22:10.437 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-03-20T11:22:10.437 DEBUG:teuthology.orchestra.run.vm07:> sudo mkdir -p /var/lib/ceph/mon/c2-a 2026-03-20T11:22:10.463 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster c2 --mkfs -i a --monmap /home/ubuntu/cephtest/c2.monmap --keyring /etc/ceph/c2.keyring 2026-03-20T11:22:10.563 DEBUG:teuthology.orchestra.run.vm07:> sudo chown -R ceph:ceph /var/lib/ceph/mon/c2-a 2026-03-20T11:22:10.593 DEBUG:teuthology.orchestra.run.vm07:> rm -- /home/ubuntu/cephtest/c2.monmap 2026-03-20T11:22:10.652 INFO:tasks.ceph:Starting mon daemons in cluster c2... 2026-03-20T11:22:10.652 INFO:tasks.ceph.c2.mon.a:Restarting daemon 2026-03-20T11:22:10.652 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster c2 -i a 2026-03-20T11:22:10.694 INFO:tasks.ceph.c2.mon.a:Started 2026-03-20T11:22:10.695 INFO:tasks.ceph:Starting mgr daemons in cluster c2... 2026-03-20T11:22:10.695 INFO:tasks.ceph.c2.mgr.x:Restarting daemon 2026-03-20T11:22:10.695 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster c2 -i x 2026-03-20T11:22:10.697 INFO:tasks.ceph.c2.mgr.x:Started 2026-03-20T11:22:10.697 DEBUG:tasks.ceph:set 0 configs 2026-03-20T11:22:10.697 DEBUG:teuthology.orchestra.run.vm07:> sudo ceph --cluster c2 config dump 2026-03-20T11:22:10.841 INFO:teuthology.orchestra.run.vm07.stdout:WHO MASK LEVEL OPTION VALUE RO 2026-03-20T11:22:10.852 INFO:tasks.ceph:Setting crush tunables to default 2026-03-20T11:22:10.852 DEBUG:teuthology.orchestra.run.vm07:> sudo ceph --cluster c2 osd crush tunables default 2026-03-20T11:22:10.978 INFO:teuthology.orchestra.run.vm07.stderr:adjusted tunables profile to default 2026-03-20T11:22:10.989 INFO:tasks.ceph:check_enable_crimson: False 2026-03-20T11:22:10.989 INFO:tasks.ceph:Starting osd daemons in cluster c2... 2026-03-20T11:22:10.990 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-20T11:22:10.990 DEBUG:teuthology.orchestra.run.vm07:> sudo dd if=/var/lib/ceph/osd/c2-0/fsid of=/dev/stdout 2026-03-20T11:22:11.018 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-20T11:22:11.018 DEBUG:teuthology.orchestra.run.vm07:> sudo dd if=/var/lib/ceph/osd/c2-1/fsid of=/dev/stdout 2026-03-20T11:22:11.085 DEBUG:teuthology.orchestra.run.vm07:> set -ex 2026-03-20T11:22:11.085 DEBUG:teuthology.orchestra.run.vm07:> sudo dd if=/var/lib/ceph/osd/c2-2/fsid of=/dev/stdout 2026-03-20T11:22:11.153 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-20T11:22:11.153 DEBUG:teuthology.orchestra.run.vm08:> sudo dd if=/var/lib/ceph/osd/c2-3/fsid of=/dev/stdout 2026-03-20T11:22:11.180 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-20T11:22:11.180 DEBUG:teuthology.orchestra.run.vm08:> sudo dd if=/var/lib/ceph/osd/c2-4/fsid of=/dev/stdout 2026-03-20T11:22:11.248 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-20T11:22:11.249 DEBUG:teuthology.orchestra.run.vm08:> sudo dd if=/var/lib/ceph/osd/c2-5/fsid of=/dev/stdout 2026-03-20T11:22:11.317 DEBUG:teuthology.orchestra.run.vm08:> sudo ceph --cluster c2 osd new ca96d491-87e0-4fda-91e1-9c894e1c659c 0 2026-03-20T11:22:11.482 INFO:teuthology.orchestra.run.vm08.stdout:0 2026-03-20T11:22:11.492 DEBUG:teuthology.orchestra.run.vm08:> sudo ceph --cluster c2 osd new e29a966f-7a8a-4245-8868-a0ff2670bd8f 1 2026-03-20T11:22:11.618 INFO:teuthology.orchestra.run.vm08.stdout:1 2026-03-20T11:22:11.628 DEBUG:teuthology.orchestra.run.vm08:> sudo ceph --cluster c2 osd new f3f5fdba-76f4-4476-aa5d-bdaaf2c86285 2 2026-03-20T11:22:11.746 INFO:teuthology.orchestra.run.vm08.stdout:2 2026-03-20T11:22:11.757 DEBUG:teuthology.orchestra.run.vm08:> sudo ceph --cluster c2 osd new c3bfbeef-424f-4cab-b3b5-215b977f1a70 3 2026-03-20T11:22:11.817 INFO:tasks.ceph.c2.mgr.x.vm08.stderr:/usr/lib64/python3.9/site-packages/scipy/__init__.py:73: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-03-20T11:22:11.817 INFO:tasks.ceph.c2.mgr.x.vm08.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-03-20T11:22:11.818 INFO:tasks.ceph.c2.mgr.x.vm08.stderr: from numpy import show_config as show_numpy_config 2026-03-20T11:22:11.878 INFO:teuthology.orchestra.run.vm08.stdout:3 2026-03-20T11:22:11.890 DEBUG:teuthology.orchestra.run.vm08:> sudo ceph --cluster c2 osd new 0ee6cde1-7991-4b22-a1cf-efde19937764 4 2026-03-20T11:22:12.008 INFO:teuthology.orchestra.run.vm08.stdout:4 2026-03-20T11:22:12.018 DEBUG:teuthology.orchestra.run.vm08:> sudo ceph --cluster c2 osd new 9ef2428f-17bf-44b1-a389-c620b6ff8076 5 2026-03-20T11:22:12.144 INFO:teuthology.orchestra.run.vm08.stdout:5 2026-03-20T11:22:12.155 INFO:tasks.ceph.c2.osd.0:Restarting daemon 2026-03-20T11:22:12.155 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster c2 -i 0 2026-03-20T11:22:12.157 INFO:tasks.ceph.c2.osd.0:Started 2026-03-20T11:22:12.157 INFO:tasks.ceph.c2.osd.1:Restarting daemon 2026-03-20T11:22:12.157 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster c2 -i 1 2026-03-20T11:22:12.158 INFO:tasks.ceph.c2.osd.1:Started 2026-03-20T11:22:12.158 INFO:tasks.ceph.c2.osd.2:Restarting daemon 2026-03-20T11:22:12.158 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster c2 -i 2 2026-03-20T11:22:12.160 INFO:tasks.ceph.c2.osd.2:Started 2026-03-20T11:22:12.160 INFO:tasks.ceph.c2.osd.3:Restarting daemon 2026-03-20T11:22:12.160 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster c2 -i 3 2026-03-20T11:22:12.162 INFO:tasks.ceph.c2.osd.3:Started 2026-03-20T11:22:12.162 INFO:tasks.ceph.c2.osd.4:Restarting daemon 2026-03-20T11:22:12.162 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster c2 -i 4 2026-03-20T11:22:12.164 INFO:tasks.ceph.c2.osd.4:Started 2026-03-20T11:22:12.164 INFO:tasks.ceph.c2.osd.5:Restarting daemon 2026-03-20T11:22:12.164 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster c2 -i 5 2026-03-20T11:22:12.166 INFO:tasks.ceph.c2.osd.5:Started 2026-03-20T11:22:12.166 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd dump --format=json 2026-03-20T11:22:12.306 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:12.307 INFO:teuthology.orchestra.run.vm07.stdout:{"epoch":8,"fsid":"9e939ad0-eb90-4072-8220-014224b87802","created":"2026-03-20T11:22:10.789651+0000","modified":"2026-03-20T11:22:12.140845+0000","last_up_change":"0.000000","last_in_change":"2026-03-20T11:22:12.140845+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":6,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"ca96d491-87e0-4fda-91e1-9c894e1c659c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"e29a966f-7a8a-4245-8868-a0ff2670bd8f","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"f3f5fdba-76f4-4476-aa5d-bdaaf2c86285","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"c3bfbeef-424f-4cab-b3b5-215b977f1a70","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":4,"uuid":"0ee6cde1-7991-4b22-a1cf-efde19937764","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":5,"uuid":"9ef2428f-17bf-44b1-a389-c620b6ff8076","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-20T11:22:12.317 INFO:tasks.ceph.ceph_manager.c2:[] 2026-03-20T11:22:12.318 INFO:tasks.ceph:Waiting for OSDs to come up 2026-03-20T11:22:12.344 INFO:tasks.ceph.c2.osd.4.vm08.stderr:2026-03-20T11:22:12.345+0000 7f42fa320900 -1 Falling back to public interface 2026-03-20T11:22:12.364 INFO:tasks.ceph.c2.osd.2.vm07.stderr:2026-03-20T11:22:12.364+0000 7f11f86ce900 -1 Falling back to public interface 2026-03-20T11:22:12.375 INFO:tasks.ceph.c2.osd.3.vm08.stderr:2026-03-20T11:22:12.376+0000 7ff5e51d3900 -1 Falling back to public interface 2026-03-20T11:22:12.375 INFO:tasks.ceph.c2.osd.5.vm08.stderr:2026-03-20T11:22:12.377+0000 7f044b097900 -1 Falling back to public interface 2026-03-20T11:22:12.379 INFO:tasks.ceph.c2.osd.1.vm07.stderr:2026-03-20T11:22:12.380+0000 7f4428b4d900 -1 Falling back to public interface 2026-03-20T11:22:12.383 INFO:tasks.ceph.c2.osd.0.vm07.stderr:2026-03-20T11:22:12.384+0000 7fa5c0bb6900 -1 Falling back to public interface 2026-03-20T11:22:12.500 INFO:tasks.ceph.c2.osd.4.vm08.stderr:2026-03-20T11:22:12.501+0000 7f42fa320900 -1 osd.4 0 log_to_monitors true 2026-03-20T11:22:12.513 INFO:tasks.ceph.c2.osd.2.vm07.stderr:2026-03-20T11:22:12.513+0000 7f11f86ce900 -1 osd.2 0 log_to_monitors true 2026-03-20T11:22:12.533 INFO:tasks.ceph.c2.osd.1.vm07.stderr:2026-03-20T11:22:12.534+0000 7f4428b4d900 -1 osd.1 0 log_to_monitors true 2026-03-20T11:22:12.540 INFO:tasks.ceph.c2.osd.0.vm07.stderr:2026-03-20T11:22:12.541+0000 7fa5c0bb6900 -1 osd.0 0 log_to_monitors true 2026-03-20T11:22:12.549 INFO:tasks.ceph.c2.osd.5.vm08.stderr:2026-03-20T11:22:12.550+0000 7f044b097900 -1 osd.5 0 log_to_monitors true 2026-03-20T11:22:12.549 INFO:tasks.ceph.c2.osd.3.vm08.stderr:2026-03-20T11:22:12.551+0000 7ff5e51d3900 -1 osd.3 0 log_to_monitors true 2026-03-20T11:22:12.922 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster c2 osd dump --format=json 2026-03-20T11:22:13.036 INFO:teuthology.misc.health.vm07.stdout: 2026-03-20T11:22:13.036 INFO:teuthology.misc.health.vm07.stdout:{"epoch":8,"fsid":"9e939ad0-eb90-4072-8220-014224b87802","created":"2026-03-20T11:22:10.789651+0000","modified":"2026-03-20T11:22:12.140845+0000","last_up_change":"0.000000","last_in_change":"2026-03-20T11:22:12.140845+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":6,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"ca96d491-87e0-4fda-91e1-9c894e1c659c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"e29a966f-7a8a-4245-8868-a0ff2670bd8f","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"f3f5fdba-76f4-4476-aa5d-bdaaf2c86285","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"c3bfbeef-424f-4cab-b3b5-215b977f1a70","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":4,"uuid":"0ee6cde1-7991-4b22-a1cf-efde19937764","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":5,"uuid":"9ef2428f-17bf-44b1-a389-c620b6ff8076","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-20T11:22:13.045 DEBUG:teuthology.misc:0 of 6 OSDs are up 2026-03-20T11:22:14.150 INFO:tasks.ceph.c2.osd.0.vm07.stderr:2026-03-20T11:22:14.151+0000 7fa5bd357640 -1 osd.0 0 waiting for initial osdmap 2026-03-20T11:22:14.151 INFO:tasks.ceph.c2.osd.5.vm08.stderr:2026-03-20T11:22:14.152+0000 7f044783a640 -1 osd.5 0 waiting for initial osdmap 2026-03-20T11:22:14.151 INFO:tasks.ceph.c2.osd.1.vm07.stderr:2026-03-20T11:22:14.152+0000 7f44252ee640 -1 osd.1 0 waiting for initial osdmap 2026-03-20T11:22:14.151 INFO:tasks.ceph.c2.osd.3.vm08.stderr:2026-03-20T11:22:14.152+0000 7ff5e1974640 -1 osd.3 0 waiting for initial osdmap 2026-03-20T11:22:14.151 INFO:tasks.ceph.c2.osd.4.vm08.stderr:2026-03-20T11:22:14.153+0000 7f42f6ac1640 -1 osd.4 0 waiting for initial osdmap 2026-03-20T11:22:14.151 INFO:tasks.ceph.c2.osd.2.vm07.stderr:2026-03-20T11:22:14.152+0000 7f11f465d640 -1 osd.2 0 waiting for initial osdmap 2026-03-20T11:22:14.161 INFO:tasks.ceph.c2.osd.4.vm08.stderr:2026-03-20T11:22:14.163+0000 7f42f10b4640 -1 osd.4 10 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-03-20T11:22:14.162 INFO:tasks.ceph.c2.osd.3.vm08.stderr:2026-03-20T11:22:14.163+0000 7ff5dbf67640 -1 osd.3 10 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-03-20T11:22:14.165 INFO:tasks.ceph.c2.osd.5.vm08.stderr:2026-03-20T11:22:14.167+0000 7f0441e2d640 -1 osd.5 10 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-03-20T11:22:14.166 INFO:tasks.ceph.c2.osd.1.vm07.stderr:2026-03-20T11:22:14.167+0000 7f441f8e1640 -1 osd.1 10 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-03-20T11:22:14.166 INFO:tasks.ceph.c2.osd.2.vm07.stderr:2026-03-20T11:22:14.167+0000 7f11ef462640 -1 osd.2 10 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-03-20T11:22:14.166 INFO:tasks.ceph.c2.osd.0.vm07.stderr:2026-03-20T11:22:14.167+0000 7fa5b794a640 -1 osd.0 10 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-03-20T11:22:14.491 INFO:tasks.ceph.c2.mgr.x.vm08.stderr:2026-03-20T11:22:14.492+0000 7fce962c1640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-03-20T11:22:19.648 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster c2 osd dump --format=json 2026-03-20T11:22:19.834 INFO:teuthology.misc.health.vm07.stdout: 2026-03-20T11:22:19.834 INFO:teuthology.misc.health.vm07.stdout:{"epoch":14,"fsid":"9e939ad0-eb90-4072-8220-014224b87802","created":"2026-03-20T11:22:10.789651+0000","modified":"2026-03-20T11:22:18.394607+0000","last_up_change":"2026-03-20T11:22:15.149027+0000","last_in_change":"2026-03-20T11:22:12.140845+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":6,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-03-20T11:22:15.569447+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":6.059999942779541,"score_stable":6.059999942779541,"optimal_score":0.33000001311302185,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca96d491-87e0-4fda-91e1-9c894e1c659c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6816","nonce":3817513893},{"type":"v1","addr":"192.168.123.107:6817","nonce":3817513893}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6818","nonce":3817513893},{"type":"v1","addr":"192.168.123.107:6819","nonce":3817513893}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6822","nonce":3817513893},{"type":"v1","addr":"192.168.123.107:6823","nonce":3817513893}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6820","nonce":3817513893},{"type":"v1","addr":"192.168.123.107:6821","nonce":3817513893}]},"public_addr":"192.168.123.107:6817/3817513893","cluster_addr":"192.168.123.107:6819/3817513893","heartbeat_back_addr":"192.168.123.107:6823/3817513893","heartbeat_front_addr":"192.168.123.107:6821/3817513893","state":["exists","up"]},{"osd":1,"uuid":"e29a966f-7a8a-4245-8868-a0ff2670bd8f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6808","nonce":2173893926},{"type":"v1","addr":"192.168.123.107:6809","nonce":2173893926}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6810","nonce":2173893926},{"type":"v1","addr":"192.168.123.107:6811","nonce":2173893926}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6814","nonce":2173893926},{"type":"v1","addr":"192.168.123.107:6815","nonce":2173893926}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6812","nonce":2173893926},{"type":"v1","addr":"192.168.123.107:6813","nonce":2173893926}]},"public_addr":"192.168.123.107:6809/2173893926","cluster_addr":"192.168.123.107:6811/2173893926","heartbeat_back_addr":"192.168.123.107:6815/2173893926","heartbeat_front_addr":"192.168.123.107:6813/2173893926","state":["exists","up"]},{"osd":2,"uuid":"f3f5fdba-76f4-4476-aa5d-bdaaf2c86285","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6800","nonce":700456162},{"type":"v1","addr":"192.168.123.107:6801","nonce":700456162}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6802","nonce":700456162},{"type":"v1","addr":"192.168.123.107:6803","nonce":700456162}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6806","nonce":700456162},{"type":"v1","addr":"192.168.123.107:6807","nonce":700456162}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6804","nonce":700456162},{"type":"v1","addr":"192.168.123.107:6805","nonce":700456162}]},"public_addr":"192.168.123.107:6801/700456162","cluster_addr":"192.168.123.107:6803/700456162","heartbeat_back_addr":"192.168.123.107:6807/700456162","heartbeat_front_addr":"192.168.123.107:6805/700456162","state":["exists","up"]},{"osd":3,"uuid":"c3bfbeef-424f-4cab-b3b5-215b977f1a70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6808","nonce":1087427588},{"type":"v1","addr":"192.168.123.108:6809","nonce":1087427588}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6810","nonce":1087427588},{"type":"v1","addr":"192.168.123.108:6811","nonce":1087427588}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6820","nonce":1087427588},{"type":"v1","addr":"192.168.123.108:6823","nonce":1087427588}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6813","nonce":1087427588},{"type":"v1","addr":"192.168.123.108:6815","nonce":1087427588}]},"public_addr":"192.168.123.108:6809/1087427588","cluster_addr":"192.168.123.108:6811/1087427588","heartbeat_back_addr":"192.168.123.108:6823/1087427588","heartbeat_front_addr":"192.168.123.108:6815/1087427588","state":["exists","up"]},{"osd":4,"uuid":"0ee6cde1-7991-4b22-a1cf-efde19937764","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6800","nonce":3944898265},{"type":"v1","addr":"192.168.123.108:6801","nonce":3944898265}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6802","nonce":3944898265},{"type":"v1","addr":"192.168.123.108:6803","nonce":3944898265}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6806","nonce":3944898265},{"type":"v1","addr":"192.168.123.108:6807","nonce":3944898265}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6804","nonce":3944898265},{"type":"v1","addr":"192.168.123.108:6805","nonce":3944898265}]},"public_addr":"192.168.123.108:6801/3944898265","cluster_addr":"192.168.123.108:6803/3944898265","heartbeat_back_addr":"192.168.123.108:6807/3944898265","heartbeat_front_addr":"192.168.123.108:6805/3944898265","state":["exists","up"]},{"osd":5,"uuid":"9ef2428f-17bf-44b1-a389-c620b6ff8076","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6812","nonce":2879676188},{"type":"v1","addr":"192.168.123.108:6814","nonce":2879676188}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6816","nonce":2879676188},{"type":"v1","addr":"192.168.123.108:6817","nonce":2879676188}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6821","nonce":2879676188},{"type":"v1","addr":"192.168.123.108:6822","nonce":2879676188}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6818","nonce":2879676188},{"type":"v1","addr":"192.168.123.108:6819","nonce":2879676188}]},"public_addr":"192.168.123.108:6814/2879676188","cluster_addr":"192.168.123.108:6817/2879676188","heartbeat_back_addr":"192.168.123.108:6822/2879676188","heartbeat_front_addr":"192.168.123.108:6819/2879676188","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:22:13.524590+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:22:13.539938+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:22:13.566819+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:22:13.574099+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:22:13.520043+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:22:13.532137+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-20T11:22:19.846 DEBUG:teuthology.misc:6 of 6 OSDs are up 2026-03-20T11:22:19.846 INFO:tasks.ceph:Creating RBD pool 2026-03-20T11:22:19.846 DEBUG:teuthology.orchestra.run.vm07:> sudo ceph --cluster c2 osd pool create rbd 8 2026-03-20T11:22:20.404 INFO:teuthology.orchestra.run.vm07.stderr:pool 'rbd' created 2026-03-20T11:22:20.427 DEBUG:teuthology.orchestra.run.vm07:> rbd --cluster c2 pool init rbd 2026-03-20T11:22:20.467 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:22:20.467 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:22:23.437 INFO:tasks.ceph:Starting mds daemons in cluster c2... 2026-03-20T11:22:23.438 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 config log 1 --format=json 2026-03-20T11:22:23.438 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-03-20T11:22:23.703 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:23.717 INFO:teuthology.orchestra.run.vm07.stdout:[{"version":1,"timestamp":"0.000000","name":"","changes":[]}] 2026-03-20T11:22:23.717 INFO:tasks.ceph_manager:config epoch is 1 2026-03-20T11:22:23.717 INFO:tasks.ceph:Waiting until c2 daemons up and pgs clean... 2026-03-20T11:22:23.717 INFO:tasks.ceph.ceph_manager.c2:waiting for mgr available 2026-03-20T11:22:23.717 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 mgr dump --format=json 2026-03-20T11:22:23.963 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:23.978 INFO:teuthology.orchestra.run.vm07.stdout:{"epoch":5,"flags":0,"active_gid":4103,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6824","nonce":2795511785},{"type":"v1","addr":"192.168.123.108:6825","nonce":2795511785}]},"active_addr":"192.168.123.108:6825/2795511785","active_change":"2026-03-20T11:22:13.475294+0000","active_mgr_features":4544132024016699391,"available":true,"standbys":[],"modules":["iostat","nfs"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"certificate_automated_rotation_enabled":{"name":"certificate_automated_rotation_enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"This flag controls whether cephadm automatically rotates certificates upon expiration.","long_desc":"","tags":[],"see_also":[]},"certificate_check_debug_mode":{"name":"certificate_check_debug_mode","type":"bool","level":"dev","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"FOR TESTING ONLY: This flag forces the certificate check instead of waiting for certificate_check_period.","long_desc":"","tags":[],"see_also":[]},"certificate_check_period":{"name":"certificate_check_period","type":"int","level":"advanced","flags":0,"default_value":"1","min":"0","max":"30","enum_allowed":[],"desc":"Specifies how often (in days) the certificate should be checked for validity.","long_desc":"","tags":[],"see_also":[]},"certificate_duration_days":{"name":"certificate_duration_days","type":"int","level":"advanced","flags":0,"default_value":"1095","min":"90","max":"3650","enum_allowed":[],"desc":"Specifies the duration of self certificates generated and signed by cephadm root CA","long_desc":"","tags":[],"see_also":[]},"certificate_renewal_threshold_days":{"name":"certificate_renewal_threshold_days","type":"int","level":"advanced","flags":0,"default_value":"30","min":"10","max":"90","enum_allowed":[],"desc":"Specifies the lead time in days to initiate certificate renewal before expiration.","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.28.1","min":"","max":"","enum_allowed":[],"desc":"Alertmanager container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"Elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:12.3.1","min":"","max":"","enum_allowed":[],"desc":"Grafana container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"Haproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:sclorg-nginx-126","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.9.1","min":"","max":"","enum_allowed":[],"desc":"Node exporter container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.5","min":"","max":"","enum_allowed":[],"desc":"Nvmeof container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"Oauth2 proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v3.6.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba_metrics":{"name":"container_image_samba_metrics","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-metrics:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba metrics container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"Snmp gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"stray_daemon_check_interval":{"name":"stray_daemon_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"how frequently cephadm should check for the presence of stray daemons","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROM_ALERT_CREDENTIAL_CACHE_TTL":{"name":"PROM_ALERT_CREDENTIAL_CACHE_TTL","type":"int","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_HOSTNAME_PER_DAEMON":{"name":"RGW_HOSTNAME_PER_DAEMON","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crypto_caller":{"name":"crypto_caller","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sso_oauth2":{"name":"sso_oauth2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"prometheus_tls_secret_name":{"name":"prometheus_tls_secret_name","type":"str","level":"advanced","flags":0,"default_value":"rook-ceph-prometheus-server-tls","min":"","max":"","enum_allowed":[],"desc":"name of tls secret in k8s for prometheus","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_cloning":{"name":"pause_cloning","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_purging":{"name":"pause_purging","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous subvolume purge threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"tentacle":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"192.168.123.108:0","nonce":868785440}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"192.168.123.108:0","nonce":2748129076}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"192.168.123.108:0","nonce":4288472694}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"192.168.123.108:0","nonce":4116296868}]}]} 2026-03-20T11:22:23.979 INFO:tasks.ceph.ceph_manager.c2:mgr available! 2026-03-20T11:22:23.979 INFO:tasks.ceph.ceph_manager.c2:waiting for all up 2026-03-20T11:22:23.979 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd dump --format=json 2026-03-20T11:22:24.200 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:24.200 INFO:teuthology.orchestra.run.vm07.stdout:{"epoch":18,"fsid":"9e939ad0-eb90-4072-8220-014224b87802","created":"2026-03-20T11:22:10.789651+0000","modified":"2026-03-20T11:22:23.418722+0000","last_up_change":"2026-03-20T11:22:15.149027+0000","last_in_change":"2026-03-20T11:22:12.140845+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":6,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-03-20T11:22:15.569447+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":6.059999942779541,"score_stable":6.059999942779541,"optimal_score":0.33000001311302185,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-03-20T11:22:20.045217+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":18,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca96d491-87e0-4fda-91e1-9c894e1c659c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6816","nonce":3817513893},{"type":"v1","addr":"192.168.123.107:6817","nonce":3817513893}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6818","nonce":3817513893},{"type":"v1","addr":"192.168.123.107:6819","nonce":3817513893}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6822","nonce":3817513893},{"type":"v1","addr":"192.168.123.107:6823","nonce":3817513893}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6820","nonce":3817513893},{"type":"v1","addr":"192.168.123.107:6821","nonce":3817513893}]},"public_addr":"192.168.123.107:6817/3817513893","cluster_addr":"192.168.123.107:6819/3817513893","heartbeat_back_addr":"192.168.123.107:6823/3817513893","heartbeat_front_addr":"192.168.123.107:6821/3817513893","state":["exists","up"]},{"osd":1,"uuid":"e29a966f-7a8a-4245-8868-a0ff2670bd8f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6808","nonce":2173893926},{"type":"v1","addr":"192.168.123.107:6809","nonce":2173893926}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6810","nonce":2173893926},{"type":"v1","addr":"192.168.123.107:6811","nonce":2173893926}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6814","nonce":2173893926},{"type":"v1","addr":"192.168.123.107:6815","nonce":2173893926}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6812","nonce":2173893926},{"type":"v1","addr":"192.168.123.107:6813","nonce":2173893926}]},"public_addr":"192.168.123.107:6809/2173893926","cluster_addr":"192.168.123.107:6811/2173893926","heartbeat_back_addr":"192.168.123.107:6815/2173893926","heartbeat_front_addr":"192.168.123.107:6813/2173893926","state":["exists","up"]},{"osd":2,"uuid":"f3f5fdba-76f4-4476-aa5d-bdaaf2c86285","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6800","nonce":700456162},{"type":"v1","addr":"192.168.123.107:6801","nonce":700456162}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6802","nonce":700456162},{"type":"v1","addr":"192.168.123.107:6803","nonce":700456162}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6806","nonce":700456162},{"type":"v1","addr":"192.168.123.107:6807","nonce":700456162}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6804","nonce":700456162},{"type":"v1","addr":"192.168.123.107:6805","nonce":700456162}]},"public_addr":"192.168.123.107:6801/700456162","cluster_addr":"192.168.123.107:6803/700456162","heartbeat_back_addr":"192.168.123.107:6807/700456162","heartbeat_front_addr":"192.168.123.107:6805/700456162","state":["exists","up"]},{"osd":3,"uuid":"c3bfbeef-424f-4cab-b3b5-215b977f1a70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6808","nonce":1087427588},{"type":"v1","addr":"192.168.123.108:6809","nonce":1087427588}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6810","nonce":1087427588},{"type":"v1","addr":"192.168.123.108:6811","nonce":1087427588}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6820","nonce":1087427588},{"type":"v1","addr":"192.168.123.108:6823","nonce":1087427588}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6813","nonce":1087427588},{"type":"v1","addr":"192.168.123.108:6815","nonce":1087427588}]},"public_addr":"192.168.123.108:6809/1087427588","cluster_addr":"192.168.123.108:6811/1087427588","heartbeat_back_addr":"192.168.123.108:6823/1087427588","heartbeat_front_addr":"192.168.123.108:6815/1087427588","state":["exists","up"]},{"osd":4,"uuid":"0ee6cde1-7991-4b22-a1cf-efde19937764","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6800","nonce":3944898265},{"type":"v1","addr":"192.168.123.108:6801","nonce":3944898265}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6802","nonce":3944898265},{"type":"v1","addr":"192.168.123.108:6803","nonce":3944898265}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6806","nonce":3944898265},{"type":"v1","addr":"192.168.123.108:6807","nonce":3944898265}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6804","nonce":3944898265},{"type":"v1","addr":"192.168.123.108:6805","nonce":3944898265}]},"public_addr":"192.168.123.108:6801/3944898265","cluster_addr":"192.168.123.108:6803/3944898265","heartbeat_back_addr":"192.168.123.108:6807/3944898265","heartbeat_front_addr":"192.168.123.108:6805/3944898265","state":["exists","up"]},{"osd":5,"uuid":"9ef2428f-17bf-44b1-a389-c620b6ff8076","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6812","nonce":2879676188},{"type":"v1","addr":"192.168.123.108:6814","nonce":2879676188}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6816","nonce":2879676188},{"type":"v1","addr":"192.168.123.108:6817","nonce":2879676188}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6821","nonce":2879676188},{"type":"v1","addr":"192.168.123.108:6822","nonce":2879676188}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6818","nonce":2879676188},{"type":"v1","addr":"192.168.123.108:6819","nonce":2879676188}]},"public_addr":"192.168.123.108:6814/2879676188","cluster_addr":"192.168.123.108:6817/2879676188","heartbeat_back_addr":"192.168.123.108:6822/2879676188","heartbeat_front_addr":"192.168.123.108:6819/2879676188","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:22:13.524590+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:22:13.539938+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:22:13.566819+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:22:13.574099+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:22:13.520043+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:22:13.532137+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-20T11:22:24.214 INFO:tasks.ceph.ceph_manager.c2:all up! 2026-03-20T11:22:24.215 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd dump --format=json 2026-03-20T11:22:24.438 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:24.438 INFO:teuthology.orchestra.run.vm07.stdout:{"epoch":18,"fsid":"9e939ad0-eb90-4072-8220-014224b87802","created":"2026-03-20T11:22:10.789651+0000","modified":"2026-03-20T11:22:23.418722+0000","last_up_change":"2026-03-20T11:22:15.149027+0000","last_in_change":"2026-03-20T11:22:12.140845+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":6,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-03-20T11:22:15.569447+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":6.059999942779541,"score_stable":6.059999942779541,"optimal_score":0.33000001311302185,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-03-20T11:22:20.045217+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":18,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca96d491-87e0-4fda-91e1-9c894e1c659c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6816","nonce":3817513893},{"type":"v1","addr":"192.168.123.107:6817","nonce":3817513893}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6818","nonce":3817513893},{"type":"v1","addr":"192.168.123.107:6819","nonce":3817513893}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6822","nonce":3817513893},{"type":"v1","addr":"192.168.123.107:6823","nonce":3817513893}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6820","nonce":3817513893},{"type":"v1","addr":"192.168.123.107:6821","nonce":3817513893}]},"public_addr":"192.168.123.107:6817/3817513893","cluster_addr":"192.168.123.107:6819/3817513893","heartbeat_back_addr":"192.168.123.107:6823/3817513893","heartbeat_front_addr":"192.168.123.107:6821/3817513893","state":["exists","up"]},{"osd":1,"uuid":"e29a966f-7a8a-4245-8868-a0ff2670bd8f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6808","nonce":2173893926},{"type":"v1","addr":"192.168.123.107:6809","nonce":2173893926}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6810","nonce":2173893926},{"type":"v1","addr":"192.168.123.107:6811","nonce":2173893926}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6814","nonce":2173893926},{"type":"v1","addr":"192.168.123.107:6815","nonce":2173893926}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6812","nonce":2173893926},{"type":"v1","addr":"192.168.123.107:6813","nonce":2173893926}]},"public_addr":"192.168.123.107:6809/2173893926","cluster_addr":"192.168.123.107:6811/2173893926","heartbeat_back_addr":"192.168.123.107:6815/2173893926","heartbeat_front_addr":"192.168.123.107:6813/2173893926","state":["exists","up"]},{"osd":2,"uuid":"f3f5fdba-76f4-4476-aa5d-bdaaf2c86285","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6800","nonce":700456162},{"type":"v1","addr":"192.168.123.107:6801","nonce":700456162}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6802","nonce":700456162},{"type":"v1","addr":"192.168.123.107:6803","nonce":700456162}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6806","nonce":700456162},{"type":"v1","addr":"192.168.123.107:6807","nonce":700456162}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.107:6804","nonce":700456162},{"type":"v1","addr":"192.168.123.107:6805","nonce":700456162}]},"public_addr":"192.168.123.107:6801/700456162","cluster_addr":"192.168.123.107:6803/700456162","heartbeat_back_addr":"192.168.123.107:6807/700456162","heartbeat_front_addr":"192.168.123.107:6805/700456162","state":["exists","up"]},{"osd":3,"uuid":"c3bfbeef-424f-4cab-b3b5-215b977f1a70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6808","nonce":1087427588},{"type":"v1","addr":"192.168.123.108:6809","nonce":1087427588}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6810","nonce":1087427588},{"type":"v1","addr":"192.168.123.108:6811","nonce":1087427588}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6820","nonce":1087427588},{"type":"v1","addr":"192.168.123.108:6823","nonce":1087427588}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6813","nonce":1087427588},{"type":"v1","addr":"192.168.123.108:6815","nonce":1087427588}]},"public_addr":"192.168.123.108:6809/1087427588","cluster_addr":"192.168.123.108:6811/1087427588","heartbeat_back_addr":"192.168.123.108:6823/1087427588","heartbeat_front_addr":"192.168.123.108:6815/1087427588","state":["exists","up"]},{"osd":4,"uuid":"0ee6cde1-7991-4b22-a1cf-efde19937764","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6800","nonce":3944898265},{"type":"v1","addr":"192.168.123.108:6801","nonce":3944898265}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6802","nonce":3944898265},{"type":"v1","addr":"192.168.123.108:6803","nonce":3944898265}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6806","nonce":3944898265},{"type":"v1","addr":"192.168.123.108:6807","nonce":3944898265}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6804","nonce":3944898265},{"type":"v1","addr":"192.168.123.108:6805","nonce":3944898265}]},"public_addr":"192.168.123.108:6801/3944898265","cluster_addr":"192.168.123.108:6803/3944898265","heartbeat_back_addr":"192.168.123.108:6807/3944898265","heartbeat_front_addr":"192.168.123.108:6805/3944898265","state":["exists","up"]},{"osd":5,"uuid":"9ef2428f-17bf-44b1-a389-c620b6ff8076","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6812","nonce":2879676188},{"type":"v1","addr":"192.168.123.108:6814","nonce":2879676188}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6816","nonce":2879676188},{"type":"v1","addr":"192.168.123.108:6817","nonce":2879676188}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6821","nonce":2879676188},{"type":"v1","addr":"192.168.123.108:6822","nonce":2879676188}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6818","nonce":2879676188},{"type":"v1","addr":"192.168.123.108:6819","nonce":2879676188}]},"public_addr":"192.168.123.108:6814/2879676188","cluster_addr":"192.168.123.108:6817/2879676188","heartbeat_back_addr":"192.168.123.108:6822/2879676188","heartbeat_front_addr":"192.168.123.108:6819/2879676188","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:22:13.524590+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:22:13.539938+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:22:13.566819+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:22:13.574099+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:22:13.520043+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-20T11:22:13.532137+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-20T11:22:24.454 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 tell osd.0 flush_pg_stats 2026-03-20T11:22:24.454 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 tell osd.1 flush_pg_stats 2026-03-20T11:22:24.454 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 tell osd.2 flush_pg_stats 2026-03-20T11:22:24.454 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 tell osd.3 flush_pg_stats 2026-03-20T11:22:24.454 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 tell osd.4 flush_pg_stats 2026-03-20T11:22:24.454 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 tell osd.5 flush_pg_stats 2026-03-20T11:22:24.637 INFO:teuthology.orchestra.run.vm07.stdout:47244640259 2026-03-20T11:22:24.637 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.1 2026-03-20T11:22:24.639 INFO:teuthology.orchestra.run.vm07.stdout:47244640259 2026-03-20T11:22:24.640 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.5 2026-03-20T11:22:24.643 INFO:teuthology.orchestra.run.vm07.stdout:47244640259 2026-03-20T11:22:24.645 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.3 2026-03-20T11:22:24.646 INFO:teuthology.orchestra.run.vm07.stdout:47244640259 2026-03-20T11:22:24.646 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.4 2026-03-20T11:22:24.659 INFO:teuthology.orchestra.run.vm07.stdout:47244640259 2026-03-20T11:22:24.659 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.2 2026-03-20T11:22:24.676 INFO:teuthology.orchestra.run.vm07.stdout:47244640260 2026-03-20T11:22:24.676 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.0 2026-03-20T11:22:24.922 INFO:teuthology.orchestra.run.vm07.stdout:47244640258 2026-03-20T11:22:24.955 INFO:tasks.ceph.ceph_manager.c2:need seq 47244640259 got 47244640258 for osd.1 2026-03-20T11:22:24.979 INFO:teuthology.orchestra.run.vm07.stdout:47244640258 2026-03-20T11:22:24.983 INFO:teuthology.orchestra.run.vm07.stdout:47244640258 2026-03-20T11:22:24.993 INFO:teuthology.orchestra.run.vm07.stdout:47244640259 2026-03-20T11:22:24.995 INFO:tasks.ceph.ceph_manager.c2:need seq 47244640259 got 47244640258 for osd.3 2026-03-20T11:22:25.003 INFO:tasks.ceph.ceph_manager.c2:need seq 47244640259 got 47244640258 for osd.5 2026-03-20T11:22:25.013 INFO:tasks.ceph.ceph_manager.c2:need seq 47244640260 got 47244640259 for osd.0 2026-03-20T11:22:25.025 INFO:teuthology.orchestra.run.vm07.stdout:47244640258 2026-03-20T11:22:25.028 INFO:teuthology.orchestra.run.vm07.stdout:47244640258 2026-03-20T11:22:25.038 INFO:tasks.ceph.ceph_manager.c2:need seq 47244640259 got 47244640258 for osd.4 2026-03-20T11:22:25.045 INFO:tasks.ceph.ceph_manager.c2:need seq 47244640259 got 47244640258 for osd.2 2026-03-20T11:22:25.956 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.1 2026-03-20T11:22:25.995 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.3 2026-03-20T11:22:26.004 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.5 2026-03-20T11:22:26.014 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.0 2026-03-20T11:22:26.039 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.4 2026-03-20T11:22:26.046 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.2 2026-03-20T11:22:26.206 INFO:teuthology.orchestra.run.vm07.stdout:47244640258 2026-03-20T11:22:26.231 INFO:tasks.ceph.ceph_manager.c2:need seq 47244640259 got 47244640258 for osd.1 2026-03-20T11:22:26.265 INFO:teuthology.orchestra.run.vm07.stdout:47244640258 2026-03-20T11:22:26.289 INFO:tasks.ceph.ceph_manager.c2:need seq 47244640259 got 47244640258 for osd.3 2026-03-20T11:22:26.296 INFO:teuthology.orchestra.run.vm07.stdout:47244640259 2026-03-20T11:22:26.315 INFO:tasks.ceph.ceph_manager.c2:need seq 47244640260 got 47244640259 for osd.0 2026-03-20T11:22:26.331 INFO:teuthology.orchestra.run.vm07.stdout:47244640258 2026-03-20T11:22:26.347 INFO:teuthology.orchestra.run.vm07.stdout:47244640258 2026-03-20T11:22:26.347 INFO:tasks.ceph.ceph_manager.c2:need seq 47244640259 got 47244640258 for osd.5 2026-03-20T11:22:26.358 INFO:teuthology.orchestra.run.vm07.stdout:47244640258 2026-03-20T11:22:26.362 INFO:tasks.ceph.ceph_manager.c2:need seq 47244640259 got 47244640258 for osd.2 2026-03-20T11:22:26.372 INFO:tasks.ceph.ceph_manager.c2:need seq 47244640259 got 47244640258 for osd.4 2026-03-20T11:22:27.232 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.1 2026-03-20T11:22:27.290 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.3 2026-03-20T11:22:27.316 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.0 2026-03-20T11:22:27.348 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.5 2026-03-20T11:22:27.363 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.2 2026-03-20T11:22:27.373 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.4 2026-03-20T11:22:27.461 INFO:teuthology.orchestra.run.vm07.stdout:47244640259 2026-03-20T11:22:27.487 INFO:tasks.ceph.ceph_manager.c2:need seq 47244640259 got 47244640259 for osd.1 2026-03-20T11:22:27.487 DEBUG:teuthology.parallel:result is None 2026-03-20T11:22:27.521 INFO:teuthology.orchestra.run.vm07.stdout:47244640259 2026-03-20T11:22:27.536 INFO:tasks.ceph.ceph_manager.c2:need seq 47244640259 got 47244640259 for osd.3 2026-03-20T11:22:27.536 DEBUG:teuthology.parallel:result is None 2026-03-20T11:22:27.594 INFO:teuthology.orchestra.run.vm07.stdout:47244640261 2026-03-20T11:22:27.611 INFO:tasks.ceph.ceph_manager.c2:need seq 47244640260 got 47244640261 for osd.0 2026-03-20T11:22:27.612 DEBUG:teuthology.parallel:result is None 2026-03-20T11:22:27.635 INFO:teuthology.orchestra.run.vm07.stdout:47244640260 2026-03-20T11:22:27.638 INFO:teuthology.orchestra.run.vm07.stdout:47244640260 2026-03-20T11:22:27.649 INFO:tasks.ceph.ceph_manager.c2:need seq 47244640259 got 47244640260 for osd.4 2026-03-20T11:22:27.649 DEBUG:teuthology.parallel:result is None 2026-03-20T11:22:27.650 INFO:tasks.ceph.ceph_manager.c2:need seq 47244640259 got 47244640260 for osd.2 2026-03-20T11:22:27.650 DEBUG:teuthology.parallel:result is None 2026-03-20T11:22:27.659 INFO:teuthology.orchestra.run.vm07.stdout:47244640260 2026-03-20T11:22:27.670 INFO:tasks.ceph.ceph_manager.c2:need seq 47244640259 got 47244640260 for osd.5 2026-03-20T11:22:27.670 DEBUG:teuthology.parallel:result is None 2026-03-20T11:22:27.670 INFO:tasks.ceph.ceph_manager.c2:waiting for clean 2026-03-20T11:22:27.670 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 pg dump --format=json 2026-03-20T11:22:27.900 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:27.900 INFO:teuthology.orchestra.run.vm07.stderr:dumped all 2026-03-20T11:22:27.911 INFO:teuthology.orchestra.run.vm07.stdout:{"pg_ready":true,"pg_map":{"version":18,"stamp":"2026-03-20T11:22:27.488251+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":191,"num_write_kb":3718,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":155,"ondisk_log_size":155,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":16,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":629145600,"kb_used":163240,"kb_used_data":2176,"kb_used_omap":47,"kb_used_meta":160848,"kb_avail":628982360,"statfs":{"total":644245094400,"available":644077936640,"internally_reserved":0,"allocated":2228224,"data_stored":1520062,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48759,"internal_metadata":164708745},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.076990"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":18,"state":"active+clean","last_fresh":"2026-03-20T11:22:23.455906+0000","last_change":"2026-03-20T11:22:23.456089+0000","last_active":"2026-03-20T11:22:23.455906+0000","last_peered":"2026-03-20T11:22:23.455906+0000","last_clean":"2026-03-20T11:22:23.455906+0000","last_became_active":"2026-03-20T11:22:21.502507+0000","last_became_peered":"2026-03-20T11:22:21.502507+0000","last_unstale":"2026-03-20T11:22:23.455906+0000","last_undegraded":"2026-03-20T11:22:23.455906+0000","last_fullsized":"2026-03-20T11:22:23.455906+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_clean_scrub_stamp":"2026-03-20T11:22:20.402924+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T19:33:31.710334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028435999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":18,"state":"active+clean","last_fresh":"2026-03-20T11:22:23.435872+0000","last_change":"2026-03-20T11:22:23.436265+0000","last_active":"2026-03-20T11:22:23.435872+0000","last_peered":"2026-03-20T11:22:23.435872+0000","last_clean":"2026-03-20T11:22:23.435872+0000","last_became_active":"2026-03-20T11:22:21.419667+0000","last_became_peered":"2026-03-20T11:22:21.419667+0000","last_unstale":"2026-03-20T11:22:23.435872+0000","last_undegraded":"2026-03-20T11:22:23.435872+0000","last_fullsized":"2026-03-20T11:22:23.435872+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_clean_scrub_stamp":"2026-03-20T11:22:20.402924+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T19:15:50.598849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00078087100000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":18,"state":"active+clean","last_fresh":"2026-03-20T11:22:23.455951+0000","last_change":"2026-03-20T11:22:23.456127+0000","last_active":"2026-03-20T11:22:23.455951+0000","last_peered":"2026-03-20T11:22:23.455951+0000","last_clean":"2026-03-20T11:22:23.455951+0000","last_became_active":"2026-03-20T11:22:21.456364+0000","last_became_peered":"2026-03-20T11:22:21.456364+0000","last_unstale":"2026-03-20T11:22:23.455951+0000","last_undegraded":"2026-03-20T11:22:23.455951+0000","last_fullsized":"2026-03-20T11:22:23.455951+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_clean_scrub_stamp":"2026-03-20T11:22:20.402924+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T19:10:58.449639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000228416,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":18,"state":"active+clean","last_fresh":"2026-03-20T11:22:23.436008+0000","last_change":"2026-03-20T11:22:23.436233+0000","last_active":"2026-03-20T11:22:23.436008+0000","last_peered":"2026-03-20T11:22:23.436008+0000","last_clean":"2026-03-20T11:22:23.436008+0000","last_became_active":"2026-03-20T11:22:21.456214+0000","last_became_peered":"2026-03-20T11:22:21.456214+0000","last_unstale":"2026-03-20T11:22:23.436008+0000","last_undegraded":"2026-03-20T11:22:23.436008+0000","last_fullsized":"2026-03-20T11:22:23.436008+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_clean_scrub_stamp":"2026-03-20T11:22:20.402924+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T11:27:09.251689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033738099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"18'2","reported_seq":22,"reported_epoch":18,"state":"active+clean","last_fresh":"2026-03-20T11:22:23.433822+0000","last_change":"2026-03-20T11:22:23.433822+0000","last_active":"2026-03-20T11:22:23.433822+0000","last_peered":"2026-03-20T11:22:23.433822+0000","last_clean":"2026-03-20T11:22:23.433822+0000","last_became_active":"2026-03-20T11:22:21.418929+0000","last_became_peered":"2026-03-20T11:22:21.418929+0000","last_unstale":"2026-03-20T11:22:23.433822+0000","last_undegraded":"2026-03-20T11:22:23.433822+0000","last_fullsized":"2026-03-20T11:22:23.433822+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_clean_scrub_stamp":"2026-03-20T11:22:20.402924+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T13:09:38.179173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000479796,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":18,"state":"active+clean","last_fresh":"2026-03-20T11:22:24.220421+0000","last_change":"2026-03-20T11:22:24.220550+0000","last_active":"2026-03-20T11:22:24.220421+0000","last_peered":"2026-03-20T11:22:24.220421+0000","last_clean":"2026-03-20T11:22:24.220421+0000","last_became_active":"2026-03-20T11:22:21.419423+0000","last_became_peered":"2026-03-20T11:22:21.419423+0000","last_unstale":"2026-03-20T11:22:24.220421+0000","last_undegraded":"2026-03-20T11:22:24.220421+0000","last_fullsized":"2026-03-20T11:22:24.220421+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_clean_scrub_stamp":"2026-03-20T11:22:20.402924+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T15:21:46.865924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027411299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":18,"state":"active+clean","last_fresh":"2026-03-20T11:22:23.456124+0000","last_change":"2026-03-20T11:22:23.456254+0000","last_active":"2026-03-20T11:22:23.456124+0000","last_peered":"2026-03-20T11:22:23.456124+0000","last_clean":"2026-03-20T11:22:23.456124+0000","last_became_active":"2026-03-20T11:22:21.419676+0000","last_became_peered":"2026-03-20T11:22:21.419676+0000","last_unstale":"2026-03-20T11:22:23.456124+0000","last_undegraded":"2026-03-20T11:22:23.456124+0000","last_fullsized":"2026-03-20T11:22:23.456124+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_clean_scrub_stamp":"2026-03-20T11:22:20.402924+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T12:20:41.839335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027636700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"16'1","reported_seq":21,"reported_epoch":18,"state":"active+clean","last_fresh":"2026-03-20T11:22:23.433474+0000","last_change":"2026-03-20T11:22:23.433566+0000","last_active":"2026-03-20T11:22:23.433474+0000","last_peered":"2026-03-20T11:22:23.433474+0000","last_clean":"2026-03-20T11:22:23.433474+0000","last_became_active":"2026-03-20T11:22:21.419577+0000","last_became_peered":"2026-03-20T11:22:21.419577+0000","last_unstale":"2026-03-20T11:22:23.433474+0000","last_undegraded":"2026-03-20T11:22:23.433474+0000","last_fullsized":"2026-03-20T11:22:23.433474+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_clean_scrub_stamp":"2026-03-20T11:22:20.402924+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T18:06:34.784092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022046199999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"13'152","reported_seq":201,"reported_epoch":18,"state":"active+clean","last_fresh":"2026-03-20T11:22:23.456023+0000","last_change":"2026-03-20T11:22:17.478897+0000","last_active":"2026-03-20T11:22:23.456023+0000","last_peered":"2026-03-20T11:22:23.456023+0000","last_clean":"2026-03-20T11:22:23.456023+0000","last_became_active":"2026-03-20T11:22:17.478458+0000","last_became_peered":"2026-03-20T11:22:17.478458+0000","last_unstale":"2026-03-20T11:22:23.456023+0000","last_undegraded":"2026-03-20T11:22:23.456023+0000","last_fullsized":"2026-03-20T11:22:23.456023+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:22:16.158524+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:22:16.158524+0000","last_clean_scrub_stamp":"2026-03-20T11:22:16.158524+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T14:34:13.071901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":189,"num_write_kb":3716,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":6},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":189,"num_write_kb":3716,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":152,"ondisk_log_size":152,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":5,"up_from":11,"seq":47244640260,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":27020,"kb_used_data":172,"kb_used_omap":7,"kb_used_meta":26808,"kb_avail":104830580,"statfs":{"total":107374182400,"available":107346513920,"internally_reserved":0,"allocated":176128,"data_stored":56567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8127,"internal_metadata":27451457},"hb_peers":[0,1,2,3,4],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":11,"seq":47244640260,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":27016,"kb_used_data":168,"kb_used_omap":8,"kb_used_meta":26807,"kb_avail":104830584,"statfs":{"total":107374182400,"available":107346518016,"internally_reserved":0,"allocated":172032,"data_stored":56548,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8777,"internal_metadata":27450807},"hb_peers":[0,1,2,3,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":11,"seq":47244640260,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":27588,"kb_used_data":748,"kb_used_omap":7,"kb_used_meta":26808,"kb_avail":104830012,"statfs":{"total":107374182400,"available":107345932288,"internally_reserved":0,"allocated":765952,"data_stored":646916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7475,"internal_metadata":27452109},"hb_peers":[0,1,2,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":11,"seq":47244640260,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":27008,"kb_used_data":168,"kb_used_omap":7,"kb_used_meta":26808,"kb_avail":104830592,"statfs":{"total":107374182400,"available":107346526208,"internally_reserved":0,"allocated":172032,"data_stored":56548,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7477,"internal_metadata":27452107},"hb_peers":[0,1,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":11,"seq":47244640260,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":27020,"kb_used_data":172,"kb_used_omap":8,"kb_used_meta":26807,"kb_avail":104830580,"statfs":{"total":107374182400,"available":107346513920,"internally_reserved":0,"allocated":176128,"data_stored":56567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8777,"internal_metadata":27450807},"hb_peers":[0,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":11,"seq":47244640261,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":27588,"kb_used_data":748,"kb_used_omap":7,"kb_used_meta":26808,"kb_avail":104830012,"statfs":{"total":107374182400,"available":107345932288,"internally_reserved":0,"allocated":765952,"data_stored":646916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8126,"internal_metadata":27451458},"hb_peers":[1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-20T11:22:27.911 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 pg dump --format=json 2026-03-20T11:22:28.100 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:28.100 INFO:teuthology.orchestra.run.vm07.stderr:dumped all 2026-03-20T11:22:28.111 INFO:teuthology.orchestra.run.vm07.stdout:{"pg_ready":true,"pg_map":{"version":18,"stamp":"2026-03-20T11:22:27.488251+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":191,"num_write_kb":3718,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":155,"ondisk_log_size":155,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":16,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":629145600,"kb_used":163240,"kb_used_data":2176,"kb_used_omap":47,"kb_used_meta":160848,"kb_avail":628982360,"statfs":{"total":644245094400,"available":644077936640,"internally_reserved":0,"allocated":2228224,"data_stored":1520062,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48759,"internal_metadata":164708745},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.076990"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":18,"state":"active+clean","last_fresh":"2026-03-20T11:22:23.455906+0000","last_change":"2026-03-20T11:22:23.456089+0000","last_active":"2026-03-20T11:22:23.455906+0000","last_peered":"2026-03-20T11:22:23.455906+0000","last_clean":"2026-03-20T11:22:23.455906+0000","last_became_active":"2026-03-20T11:22:21.502507+0000","last_became_peered":"2026-03-20T11:22:21.502507+0000","last_unstale":"2026-03-20T11:22:23.455906+0000","last_undegraded":"2026-03-20T11:22:23.455906+0000","last_fullsized":"2026-03-20T11:22:23.455906+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_clean_scrub_stamp":"2026-03-20T11:22:20.402924+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T19:33:31.710334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028435999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":18,"state":"active+clean","last_fresh":"2026-03-20T11:22:23.435872+0000","last_change":"2026-03-20T11:22:23.436265+0000","last_active":"2026-03-20T11:22:23.435872+0000","last_peered":"2026-03-20T11:22:23.435872+0000","last_clean":"2026-03-20T11:22:23.435872+0000","last_became_active":"2026-03-20T11:22:21.419667+0000","last_became_peered":"2026-03-20T11:22:21.419667+0000","last_unstale":"2026-03-20T11:22:23.435872+0000","last_undegraded":"2026-03-20T11:22:23.435872+0000","last_fullsized":"2026-03-20T11:22:23.435872+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_clean_scrub_stamp":"2026-03-20T11:22:20.402924+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T19:15:50.598849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00078087100000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":18,"state":"active+clean","last_fresh":"2026-03-20T11:22:23.455951+0000","last_change":"2026-03-20T11:22:23.456127+0000","last_active":"2026-03-20T11:22:23.455951+0000","last_peered":"2026-03-20T11:22:23.455951+0000","last_clean":"2026-03-20T11:22:23.455951+0000","last_became_active":"2026-03-20T11:22:21.456364+0000","last_became_peered":"2026-03-20T11:22:21.456364+0000","last_unstale":"2026-03-20T11:22:23.455951+0000","last_undegraded":"2026-03-20T11:22:23.455951+0000","last_fullsized":"2026-03-20T11:22:23.455951+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_clean_scrub_stamp":"2026-03-20T11:22:20.402924+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T19:10:58.449639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000228416,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":18,"state":"active+clean","last_fresh":"2026-03-20T11:22:23.436008+0000","last_change":"2026-03-20T11:22:23.436233+0000","last_active":"2026-03-20T11:22:23.436008+0000","last_peered":"2026-03-20T11:22:23.436008+0000","last_clean":"2026-03-20T11:22:23.436008+0000","last_became_active":"2026-03-20T11:22:21.456214+0000","last_became_peered":"2026-03-20T11:22:21.456214+0000","last_unstale":"2026-03-20T11:22:23.436008+0000","last_undegraded":"2026-03-20T11:22:23.436008+0000","last_fullsized":"2026-03-20T11:22:23.436008+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_clean_scrub_stamp":"2026-03-20T11:22:20.402924+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T11:27:09.251689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033738099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"18'2","reported_seq":22,"reported_epoch":18,"state":"active+clean","last_fresh":"2026-03-20T11:22:23.433822+0000","last_change":"2026-03-20T11:22:23.433822+0000","last_active":"2026-03-20T11:22:23.433822+0000","last_peered":"2026-03-20T11:22:23.433822+0000","last_clean":"2026-03-20T11:22:23.433822+0000","last_became_active":"2026-03-20T11:22:21.418929+0000","last_became_peered":"2026-03-20T11:22:21.418929+0000","last_unstale":"2026-03-20T11:22:23.433822+0000","last_undegraded":"2026-03-20T11:22:23.433822+0000","last_fullsized":"2026-03-20T11:22:23.433822+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_clean_scrub_stamp":"2026-03-20T11:22:20.402924+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T13:09:38.179173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000479796,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":18,"state":"active+clean","last_fresh":"2026-03-20T11:22:24.220421+0000","last_change":"2026-03-20T11:22:24.220550+0000","last_active":"2026-03-20T11:22:24.220421+0000","last_peered":"2026-03-20T11:22:24.220421+0000","last_clean":"2026-03-20T11:22:24.220421+0000","last_became_active":"2026-03-20T11:22:21.419423+0000","last_became_peered":"2026-03-20T11:22:21.419423+0000","last_unstale":"2026-03-20T11:22:24.220421+0000","last_undegraded":"2026-03-20T11:22:24.220421+0000","last_fullsized":"2026-03-20T11:22:24.220421+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_clean_scrub_stamp":"2026-03-20T11:22:20.402924+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T15:21:46.865924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027411299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":18,"state":"active+clean","last_fresh":"2026-03-20T11:22:23.456124+0000","last_change":"2026-03-20T11:22:23.456254+0000","last_active":"2026-03-20T11:22:23.456124+0000","last_peered":"2026-03-20T11:22:23.456124+0000","last_clean":"2026-03-20T11:22:23.456124+0000","last_became_active":"2026-03-20T11:22:21.419676+0000","last_became_peered":"2026-03-20T11:22:21.419676+0000","last_unstale":"2026-03-20T11:22:23.456124+0000","last_undegraded":"2026-03-20T11:22:23.456124+0000","last_fullsized":"2026-03-20T11:22:23.456124+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_clean_scrub_stamp":"2026-03-20T11:22:20.402924+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T12:20:41.839335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027636700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"16'1","reported_seq":21,"reported_epoch":18,"state":"active+clean","last_fresh":"2026-03-20T11:22:23.433474+0000","last_change":"2026-03-20T11:22:23.433566+0000","last_active":"2026-03-20T11:22:23.433474+0000","last_peered":"2026-03-20T11:22:23.433474+0000","last_clean":"2026-03-20T11:22:23.433474+0000","last_became_active":"2026-03-20T11:22:21.419577+0000","last_became_peered":"2026-03-20T11:22:21.419577+0000","last_unstale":"2026-03-20T11:22:23.433474+0000","last_undegraded":"2026-03-20T11:22:23.433474+0000","last_fullsized":"2026-03-20T11:22:23.433474+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:22:20.402924+0000","last_clean_scrub_stamp":"2026-03-20T11:22:20.402924+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T18:06:34.784092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022046199999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"13'152","reported_seq":201,"reported_epoch":18,"state":"active+clean","last_fresh":"2026-03-20T11:22:23.456023+0000","last_change":"2026-03-20T11:22:17.478897+0000","last_active":"2026-03-20T11:22:23.456023+0000","last_peered":"2026-03-20T11:22:23.456023+0000","last_clean":"2026-03-20T11:22:23.456023+0000","last_became_active":"2026-03-20T11:22:17.478458+0000","last_became_peered":"2026-03-20T11:22:17.478458+0000","last_unstale":"2026-03-20T11:22:23.456023+0000","last_undegraded":"2026-03-20T11:22:23.456023+0000","last_fullsized":"2026-03-20T11:22:23.456023+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-20T11:22:16.158524+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-20T11:22:16.158524+0000","last_clean_scrub_stamp":"2026-03-20T11:22:16.158524+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-21T14:34:13.071901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":189,"num_write_kb":3716,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":6},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":189,"num_write_kb":3716,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":152,"ondisk_log_size":152,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":5,"up_from":11,"seq":47244640260,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":27020,"kb_used_data":172,"kb_used_omap":7,"kb_used_meta":26808,"kb_avail":104830580,"statfs":{"total":107374182400,"available":107346513920,"internally_reserved":0,"allocated":176128,"data_stored":56567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8127,"internal_metadata":27451457},"hb_peers":[0,1,2,3,4],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":11,"seq":47244640260,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":27016,"kb_used_data":168,"kb_used_omap":8,"kb_used_meta":26807,"kb_avail":104830584,"statfs":{"total":107374182400,"available":107346518016,"internally_reserved":0,"allocated":172032,"data_stored":56548,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8777,"internal_metadata":27450807},"hb_peers":[0,1,2,3,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":11,"seq":47244640260,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":27588,"kb_used_data":748,"kb_used_omap":7,"kb_used_meta":26808,"kb_avail":104830012,"statfs":{"total":107374182400,"available":107345932288,"internally_reserved":0,"allocated":765952,"data_stored":646916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7475,"internal_metadata":27452109},"hb_peers":[0,1,2,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":11,"seq":47244640260,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":27008,"kb_used_data":168,"kb_used_omap":7,"kb_used_meta":26808,"kb_avail":104830592,"statfs":{"total":107374182400,"available":107346526208,"internally_reserved":0,"allocated":172032,"data_stored":56548,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7477,"internal_metadata":27452107},"hb_peers":[0,1,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":11,"seq":47244640260,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":27020,"kb_used_data":172,"kb_used_omap":8,"kb_used_meta":26807,"kb_avail":104830580,"statfs":{"total":107374182400,"available":107346513920,"internally_reserved":0,"allocated":176128,"data_stored":56567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8777,"internal_metadata":27450807},"hb_peers":[0,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":11,"seq":47244640261,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":27588,"kb_used_data":748,"kb_used_omap":7,"kb_used_meta":26808,"kb_avail":104830012,"statfs":{"total":107374182400,"available":107345932288,"internally_reserved":0,"allocated":765952,"data_stored":646916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8126,"internal_metadata":27451458},"hb_peers":[1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-20T11:22:28.111 INFO:tasks.ceph.ceph_manager.c2:clean! 2026-03-20T11:22:28.111 INFO:tasks.ceph:Waiting until ceph cluster c2 is healthy... 2026-03-20T11:22:28.111 INFO:tasks.ceph.ceph_manager.c2:wait_until_healthy 2026-03-20T11:22:28.112 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 health --format=json 2026-03-20T11:22:28.335 INFO:teuthology.orchestra.run.vm07.stdout: 2026-03-20T11:22:28.336 INFO:teuthology.orchestra.run.vm07.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-03-20T11:22:28.348 INFO:tasks.ceph.ceph_manager.c2:wait_until_healthy done 2026-03-20T11:22:28.348 INFO:teuthology.run_tasks:Running task rgw... 2026-03-20T11:22:28.352 DEBUG:tasks.rgw:config is {'c1.client.0': {'port': 8000}, 'c1.client.1': {'port': 8001}, 'c2.client.0': {'port': 8000}, 'c2.client.1': {'port': 8001}} 2026-03-20T11:22:28.352 DEBUG:tasks.rgw:client list is dict_keys(['c1.client.0', 'c1.client.1', 'c2.client.0', 'c2.client.1']) 2026-03-20T11:22:28.352 INFO:tasks.rgw:Creating data pools 2026-03-20T11:22:28.352 DEBUG:tasks.rgw:Obtaining remote for client c1.client.0 2026-03-20T11:22:28.352 DEBUG:teuthology.orchestra.run.vm02:> sudo ceph osd pool create default.rgw.buckets.data 64 64 --cluster c1 2026-03-20T11:22:28.612 INFO:teuthology.orchestra.run.vm02.stderr:pool 'default.rgw.buckets.data' created 2026-03-20T11:22:28.631 DEBUG:teuthology.orchestra.run.vm02:> sudo ceph osd pool application enable default.rgw.buckets.data rgw --cluster c1 2026-03-20T11:22:29.614 INFO:teuthology.orchestra.run.vm02.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.data' 2026-03-20T11:22:29.645 DEBUG:teuthology.orchestra.run.vm02:> sudo ceph osd pool create default.rgw.buckets.index 64 64 --cluster c1 2026-03-20T11:22:30.620 INFO:teuthology.orchestra.run.vm02.stderr:pool 'default.rgw.buckets.index' created 2026-03-20T11:22:30.650 DEBUG:teuthology.orchestra.run.vm02:> sudo ceph osd pool application enable default.rgw.buckets.index rgw --cluster c1 2026-03-20T11:22:31.621 INFO:teuthology.orchestra.run.vm02.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.index' 2026-03-20T11:22:31.656 DEBUG:tasks.rgw:Obtaining remote for client c1.client.1 2026-03-20T11:22:31.657 DEBUG:teuthology.orchestra.run.vm05:> sudo ceph osd pool create default.rgw.buckets.data 64 64 --cluster c1 2026-03-20T11:22:31.853 INFO:teuthology.orchestra.run.vm05.stderr:pool 'default.rgw.buckets.data' already exists 2026-03-20T11:22:31.864 DEBUG:teuthology.orchestra.run.vm05:> sudo ceph osd pool application enable default.rgw.buckets.data rgw --cluster c1 2026-03-20T11:22:32.630 INFO:teuthology.orchestra.run.vm05.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.data' 2026-03-20T11:22:32.646 DEBUG:teuthology.orchestra.run.vm05:> sudo ceph osd pool create default.rgw.buckets.index 64 64 --cluster c1 2026-03-20T11:22:32.838 INFO:teuthology.orchestra.run.vm05.stderr:pool 'default.rgw.buckets.index' already exists 2026-03-20T11:22:32.849 DEBUG:teuthology.orchestra.run.vm05:> sudo ceph osd pool application enable default.rgw.buckets.index rgw --cluster c1 2026-03-20T11:22:33.631 INFO:teuthology.orchestra.run.vm05.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.index' 2026-03-20T11:22:33.643 DEBUG:tasks.rgw:Obtaining remote for client c2.client.0 2026-03-20T11:22:33.643 DEBUG:teuthology.orchestra.run.vm07:> sudo ceph osd pool create default.rgw.buckets.data 64 64 --cluster c2 2026-03-20T11:22:34.790 INFO:teuthology.orchestra.run.vm07.stderr:pool 'default.rgw.buckets.data' created 2026-03-20T11:22:34.810 DEBUG:teuthology.orchestra.run.vm07:> sudo ceph osd pool application enable default.rgw.buckets.data rgw --cluster c2 2026-03-20T11:22:35.794 INFO:teuthology.orchestra.run.vm07.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.data' 2026-03-20T11:22:35.827 DEBUG:teuthology.orchestra.run.vm07:> sudo ceph osd pool create default.rgw.buckets.index 64 64 --cluster c2 2026-03-20T11:22:36.798 INFO:teuthology.orchestra.run.vm07.stderr:pool 'default.rgw.buckets.index' created 2026-03-20T11:22:36.818 DEBUG:teuthology.orchestra.run.vm07:> sudo ceph osd pool application enable default.rgw.buckets.index rgw --cluster c2 2026-03-20T11:22:37.800 INFO:teuthology.orchestra.run.vm07.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.index' 2026-03-20T11:22:37.832 DEBUG:tasks.rgw:Obtaining remote for client c2.client.1 2026-03-20T11:22:37.833 DEBUG:teuthology.orchestra.run.vm08:> sudo ceph osd pool create default.rgw.buckets.data 64 64 --cluster c2 2026-03-20T11:22:38.033 INFO:teuthology.orchestra.run.vm08.stderr:pool 'default.rgw.buckets.data' already exists 2026-03-20T11:22:38.044 DEBUG:teuthology.orchestra.run.vm08:> sudo ceph osd pool application enable default.rgw.buckets.data rgw --cluster c2 2026-03-20T11:22:38.807 INFO:teuthology.orchestra.run.vm08.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.data' 2026-03-20T11:22:38.819 DEBUG:teuthology.orchestra.run.vm08:> sudo ceph osd pool create default.rgw.buckets.index 64 64 --cluster c2 2026-03-20T11:22:39.010 INFO:teuthology.orchestra.run.vm08.stderr:pool 'default.rgw.buckets.index' already exists 2026-03-20T11:22:39.021 DEBUG:teuthology.orchestra.run.vm08:> sudo ceph osd pool application enable default.rgw.buckets.index rgw --cluster c2 2026-03-20T11:22:39.811 INFO:teuthology.orchestra.run.vm08.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.index' 2026-03-20T11:22:39.824 DEBUG:tasks.rgw:Pools created 2026-03-20T11:22:39.824 INFO:tasks.rgw:Configuring compression type = random 2026-03-20T11:22:39.824 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['user', 'list'] 2026-03-20T11:22:39.824 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'user', 'list'] 2026-03-20T11:22:39.824 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 user list 2026-03-20T11:22:39.859 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:22:39.859 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:22:41.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.684+0000 7faa1df2f900 20 rados->read ofs=0 len=0 2026-03-20T11:22:41.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.685+0000 7faa1df2f900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:22:41.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.685+0000 7faa1df2f900 20 realm 2026-03-20T11:22:41.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.685+0000 7faa1df2f900 20 rados->read ofs=0 len=0 2026-03-20T11:22:41.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.686+0000 7faa1df2f900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:22:41.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.686+0000 7faa1df2f900 4 RGWPeriod::init failed to init realm id : (2) No such file or directory 2026-03-20T11:22:41.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.686+0000 7faa1df2f900 20 rados->read ofs=0 len=0 2026-03-20T11:22:41.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.686+0000 7faa1df2f900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:22:41.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.686+0000 7faa1df2f900 20 rados->read ofs=0 len=0 2026-03-20T11:22:41.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.687+0000 7faa1df2f900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:22:41.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.687+0000 7faa1df2f900 20 rados->read ofs=0 len=0 2026-03-20T11:22:41.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.687+0000 7faa1df2f900 20 rados_obj.operate() r=0 bl.length=1060 2026-03-20T11:22:41.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.687+0000 7faa1df2f900 20 searching for the correct realm 2026-03-20T11:22:41.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.693+0000 7faa1df2f900 20 RGWRados::pool_iterate: got default.zonegroup. 2026-03-20T11:22:41.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.693+0000 7faa1df2f900 20 RGWRados::pool_iterate: got zonegroup_info.217c78dd-3fef-4179-ae38-a328c98d82d1 2026-03-20T11:22:41.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.693+0000 7faa1df2f900 20 RGWRados::pool_iterate: got zone_info.7ee099f5-14e5-4754-a80b-09d8b510f566 2026-03-20T11:22:41.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.693+0000 7faa1df2f900 20 RGWRados::pool_iterate: got default.zone. 2026-03-20T11:22:41.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.693+0000 7faa1df2f900 20 RGWRados::pool_iterate: got zone_names.default 2026-03-20T11:22:41.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.693+0000 7faa1df2f900 20 RGWRados::pool_iterate: got zonegroups_names.default 2026-03-20T11:22:41.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.693+0000 7faa1df2f900 20 rados->read ofs=0 len=0 2026-03-20T11:22:41.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.694+0000 7faa1df2f900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:22:41.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.694+0000 7faa1df2f900 20 rados->read ofs=0 len=0 2026-03-20T11:22:41.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.694+0000 7faa1df2f900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:22:41.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.694+0000 7faa1df2f900 20 rados->read ofs=0 len=0 2026-03-20T11:22:41.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.694+0000 7faa1df2f900 20 rados_obj.operate() r=0 bl.length=436 2026-03-20T11:22:41.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.694+0000 7faa1df2f900 20 zone default found 2026-03-20T11:22:41.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.694+0000 7faa1df2f900 4 Realm: () 2026-03-20T11:22:41.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.694+0000 7faa1df2f900 4 ZoneGroup: default (217c78dd-3fef-4179-ae38-a328c98d82d1) 2026-03-20T11:22:41.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.694+0000 7faa1df2f900 4 Zone: default (7ee099f5-14e5-4754-a80b-09d8b510f566) 2026-03-20T11:22:41.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.694+0000 7faa1df2f900 10 cannot find current period zonegroup using local zonegroup configuration 2026-03-20T11:22:41.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.694+0000 7faa1df2f900 20 zonegroup default 2026-03-20T11:22:41.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.694+0000 7faa1df2f900 20 rados->read ofs=0 len=0 2026-03-20T11:22:41.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.694+0000 7faa1df2f900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:22:41.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:41.694+0000 7faa1df2f900 20 rados->read ofs=0 len=0 2026-03-20T11:22:43.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:43.684+0000 7faa1df2f900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:22:43.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:43.684+0000 7faa1df2f900 20 rados->read ofs=0 len=0 2026-03-20T11:22:43.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:43.685+0000 7faa1df2f900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:22:43.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:43.685+0000 7faa1df2f900 20 started sync module instance, tier type = 2026-03-20T11:22:43.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:43.685+0000 7faa1df2f900 20 started zone id=7ee099f5-14e5-4754-a80b-09d8b510f566 (name=default) with tier type = 2026-03-20T11:22:45.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:45.693+0000 7faa1df2f900 20 add_watcher() i=4 2026-03-20T11:22:45.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:45.693+0000 7faa1df2f900 20 add_watcher() i=0 2026-03-20T11:22:45.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:45.694+0000 7faa1df2f900 20 add_watcher() i=6 2026-03-20T11:22:45.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:45.695+0000 7faa1df2f900 20 add_watcher() i=1 2026-03-20T11:22:45.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:45.695+0000 7faa1df2f900 20 add_watcher() i=2 2026-03-20T11:22:45.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:45.695+0000 7faa1df2f900 20 add_watcher() i=3 2026-03-20T11:22:45.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:45.698+0000 7faa1df2f900 20 add_watcher() i=7 2026-03-20T11:22:45.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:45.698+0000 7faa1df2f900 20 add_watcher() i=5 2026-03-20T11:22:45.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:45.698+0000 7faa1df2f900 2 all 8 watchers are set, enabling cache 2026-03-20T11:22:45.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:45.699+0000 7faa0affd640 5 boost::asio::awaitable, obj_version> > logback_generations::read(const DoutPrefixProvider*):446: oid=data_loggenerations_metadata not found 2026-03-20T11:22:45.698 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:45.699+0000 7faa0affd640 20 boost::asio::awaitable {anonymous}::probe_shard(const DoutPrefixProvider*, neorados::RADOS, const neorados::Object&, const neorados::IOContext&, bool&):59 probing obj=data_log.0 2026-03-20T11:22:45.698 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:45.699+0000 7faa0affd640 20 do_open: entering 2026-03-20T11:22:45.698 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:45.700+0000 7faa0a7fc640 20 boost::asio::awaitable {anonymous}::probe_shard(const DoutPrefixProvider*, neorados::RADOS, const neorados::Object&, const neorados::IOContext&, bool&):78: obj=data_log.0 does not exist 2026-03-20T11:22:45.698 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:45.700+0000 7faa0a7fc640 20 boost::asio::awaitable {anonymous}::probe_shard(const DoutPrefixProvider*, neorados::RADOS, const neorados::Object&, const neorados::IOContext&, bool&):59 probing obj=data_log.1 2026-03-20T11:22:45.698 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:45.700+0000 7faa0a7fc640 20 do_open: entering 2026-03-20T11:22:45.699 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:45.700+0000 7faa09ffb640 20 boost::asio::awaitable {anonymous}::probe_shard(const DoutPrefixProvider*, neorados::RADOS, const neorados::Object&, const neorados::IOContext&, bool&):78: obj=data_log.1 does not exist 2026-03-20T11:22:45.699 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:45.700+0000 7faa09ffb640 20 boost::asio::awaitable {anonymous}::probe_shard(const DoutPrefixProvider*, neorados::RADOS, const neorados::Object&, const neorados::IOContext&, bool&):59 probing obj=data_log.2 2026-03-20T11:22:45.699 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:45.700+0000 7faa09ffb640 20 do_open: entering 2026-03-20T11:22:45.699 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:45.700+0000 7faa1ac20640 20 boost::asio::awaitable {anonymous}::probe_shard(const DoutPrefixProvider*, neorados::RADOS, const neorados::Object&, const neorados::IOContext&, bool&):78: obj=data_log.2 does not exist 2026-03-20T11:22:45.699 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:45.700+0000 7faa1ac20640 20 boost::asio::awaitable {anonymous}::probe_shard(const DoutPrefixProvider*, neorados::RADOS, const neorados::Object&, const neorados::IOContext&, bool&):59 probing obj=data_log.3 2026-03-20T11:22:45.699 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:45.700+0000 7faa1ac20640 20 do_open: entering 2026-03-20T11:22:45.699 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:45.701+0000 7faa18c1c640 20 boost::asio::awaitable {anonymous}::probe_shard(const DoutPrefixProvider*, neorados::RADOS, const neorados::Object&, const neorados::IOContext&, bool&):78: obj=data_log.3 does not exist 2026-03-20T11:22:45.699 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:45.701+0000 7faa18c1c640 20 do_create: entering 2026-03-20T11:22:45.700 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:45.702+0000 7faa0bfff640 20 do_open: entering 2026-03-20T11:22:45.703 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:45.704+0000 7faa1df2f900 20 rgw_check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2026-03-20T11:22:45.703 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:45.704+0000 7faa1df2f900 20 rgw_check_secure_mon_conn(): mode 1 is insecure 2026-03-20T11:22:48.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:48.695+0000 7faa1df2f900 10 rgw_init_ioctx warning: failed to set recovery_priority on default.rgw.meta 2026-03-20T11:22:48.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:48.695+0000 7faa1df2f900 5 note: GC not initialized 2026-03-20T11:22:48.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:48.695+0000 7fa9beffd640 20 reqs_thread_entry: start 2026-03-20T11:22:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:48.761+0000 7faa1df2f900 20 init_complete bucket index max shards: 11 2026-03-20T11:22:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:48.761+0000 7faa1df2f900 20 Filter name: none 2026-03-20T11:22:48.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:48.761+0000 7fa9bcff9640 20 reqs_thread_entry: start 2026-03-20T11:22:48.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:48.769+0000 7faa1df2f900 20 remove_watcher() i=3 2026-03-20T11:22:48.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:48.769+0000 7faa1df2f900 2 removed watcher, disabling cache 2026-03-20T11:22:48.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:48.770+0000 7faa1df2f900 20 remove_watcher() i=0 2026-03-20T11:22:48.769 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:48.770+0000 7faa1df2f900 20 remove_watcher() i=1 2026-03-20T11:22:48.769 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:48.770+0000 7faa1df2f900 20 remove_watcher() i=2 2026-03-20T11:22:48.769 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:48.770+0000 7faa1df2f900 20 remove_watcher() i=7 2026-03-20T11:22:48.769 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:48.770+0000 7faa1df2f900 20 remove_watcher() i=5 2026-03-20T11:22:48.769 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:48.770+0000 7faa1df2f900 20 remove_watcher() i=6 2026-03-20T11:22:48.769 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:48.770+0000 7faa1df2f900 20 remove_watcher() i=4 2026-03-20T11:22:48.774 INFO:teuthology.orchestra.run.vm02.stdout:[] 2026-03-20T11:22:48.774 DEBUG:tasks.util.rgw: json result: [] 2026-03-20T11:22:48.774 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2026-03-20T11:22:48.774 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2026-03-20T11:22:48.774 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zone placement modify --rgw-zone default --placement-id default-placement --compression random 2026-03-20T11:22:48.850 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:22:48.850 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:22:48.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:48.861+0000 7fb34c5af900 20 rgw_check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2026-03-20T11:22:48.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:48.861+0000 7fb34c5af900 20 rgw_check_secure_mon_conn(): mode 1 is insecure 2026-03-20T11:22:48.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:22:48.861+0000 7fb2f8fe1640 20 reqs_thread_entry: start 2026-03-20T11:22:48.869 INFO:teuthology.orchestra.run.vm02.stdout:{"id":"7ee099f5-14e5-4754-a80b-09d8b510f566","name":"default","domain_root":"default.rgw.meta:root","control_pool":"default.rgw.control","dedup_pool":"default.rgw.dedup","gc_pool":"default.rgw.log:gc","lc_pool":"default.rgw.log:lc","log_pool":"default.rgw.log","intent_log_pool":"default.rgw.log:intent","usage_log_pool":"default.rgw.log:usage","roles_pool":"default.rgw.meta:roles","reshard_pool":"default.rgw.log:reshard","user_keys_pool":"default.rgw.meta:users.keys","user_email_pool":"default.rgw.meta:users.email","user_swift_pool":"default.rgw.meta:users.swift","user_uid_pool":"default.rgw.meta:users.uid","otp_pool":"default.rgw.otp","notif_pool":"default.rgw.log:notif","topics_pool":"default.rgw.meta:topics","account_pool":"default.rgw.meta:accounts","group_pool":"default.rgw.meta:groups","system_key":{"access_key":"","secret_key":""},"placement_pools":[{"key":"default-placement","val":{"index_pool":"default.rgw.buckets.index","storage_classes":{"STANDARD":{"data_pool":"default.rgw.buckets.data","compression_type":"random"}},"data_extra_pool":"default.rgw.buckets.non-ec","index_type":0,"inline_data":true}}],"realm_id":"","restore_pool":"default.rgw.log:restore"} 2026-03-20T11:22:48.869 DEBUG:tasks.util.rgw: json result: {'id': '7ee099f5-14e5-4754-a80b-09d8b510f566', 'name': 'default', 'domain_root': 'default.rgw.meta:root', 'control_pool': 'default.rgw.control', 'dedup_pool': 'default.rgw.dedup', 'gc_pool': 'default.rgw.log:gc', 'lc_pool': 'default.rgw.log:lc', 'log_pool': 'default.rgw.log', 'intent_log_pool': 'default.rgw.log:intent', 'usage_log_pool': 'default.rgw.log:usage', 'roles_pool': 'default.rgw.meta:roles', 'reshard_pool': 'default.rgw.log:reshard', 'user_keys_pool': 'default.rgw.meta:users.keys', 'user_email_pool': 'default.rgw.meta:users.email', 'user_swift_pool': 'default.rgw.meta:users.swift', 'user_uid_pool': 'default.rgw.meta:users.uid', 'otp_pool': 'default.rgw.otp', 'notif_pool': 'default.rgw.log:notif', 'topics_pool': 'default.rgw.meta:topics', 'account_pool': 'default.rgw.meta:accounts', 'group_pool': 'default.rgw.meta:groups', 'system_key': {'access_key': '', 'secret_key': ''}, 'placement_pools': [{'key': 'default-placement', 'val': {'index_pool': 'default.rgw.buckets.index', 'storage_classes': {'STANDARD': {'data_pool': 'default.rgw.buckets.data', 'compression_type': 'random'}}, 'data_extra_pool': 'default.rgw.buckets.non-ec', 'index_type': 0, 'inline_data': True}}], 'realm_id': '', 'restore_pool': 'default.rgw.log:restore'} 2026-03-20T11:22:48.869 INFO:tasks.util.rgw:rgwadmin: c1.client.1 : ['user', 'list'] 2026-03-20T11:22:48.869 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.1', '--cluster', 'c1', 'user', 'list'] 2026-03-20T11:22:48.869 DEBUG:teuthology.orchestra.run.vm05:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.1 --cluster c1 user list 2026-03-20T11:22:48.905 INFO:teuthology.orchestra.run.vm05.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:22:48.905 INFO:teuthology.orchestra.run.vm05.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:22:48.920 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.921+0000 7f6311b5a900 20 rados->read ofs=0 len=0 2026-03-20T11:22:48.921 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.922+0000 7f6311b5a900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:22:48.921 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.922+0000 7f6311b5a900 20 realm 2026-03-20T11:22:48.921 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.922+0000 7f6311b5a900 20 rados->read ofs=0 len=0 2026-03-20T11:22:48.921 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.922+0000 7f6311b5a900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:22:48.921 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.922+0000 7f6311b5a900 4 RGWPeriod::init failed to init realm id : (2) No such file or directory 2026-03-20T11:22:48.922 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.922+0000 7f6311b5a900 20 rados->read ofs=0 len=0 2026-03-20T11:22:48.922 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.922+0000 7f6311b5a900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:22:48.922 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.922+0000 7f6311b5a900 20 rados->read ofs=0 len=0 2026-03-20T11:22:48.922 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.923+0000 7f6311b5a900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:22:48.922 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.923+0000 7f6311b5a900 20 rados->read ofs=0 len=0 2026-03-20T11:22:48.922 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.923+0000 7f6311b5a900 20 rados_obj.operate() r=0 bl.length=1076 2026-03-20T11:22:48.922 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.923+0000 7f6311b5a900 20 searching for the correct realm 2026-03-20T11:22:48.929 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.930+0000 7f6311b5a900 20 RGWRados::pool_iterate: got default.zonegroup. 2026-03-20T11:22:48.929 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.930+0000 7f6311b5a900 20 RGWRados::pool_iterate: got zonegroup_info.217c78dd-3fef-4179-ae38-a328c98d82d1 2026-03-20T11:22:48.929 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.930+0000 7f6311b5a900 20 RGWRados::pool_iterate: got zone_info.7ee099f5-14e5-4754-a80b-09d8b510f566 2026-03-20T11:22:48.929 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.930+0000 7f6311b5a900 20 RGWRados::pool_iterate: got default.zone. 2026-03-20T11:22:48.930 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.930+0000 7f6311b5a900 20 RGWRados::pool_iterate: got zone_names.default 2026-03-20T11:22:48.930 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.930+0000 7f6311b5a900 20 RGWRados::pool_iterate: got zonegroups_names.default 2026-03-20T11:22:48.930 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.930+0000 7f6311b5a900 20 rados->read ofs=0 len=0 2026-03-20T11:22:48.930 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.930+0000 7f6311b5a900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:22:48.930 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.930+0000 7f6311b5a900 20 rados->read ofs=0 len=0 2026-03-20T11:22:48.930 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.930+0000 7f6311b5a900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:22:48.930 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.930+0000 7f6311b5a900 20 rados->read ofs=0 len=0 2026-03-20T11:22:48.930 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.931+0000 7f6311b5a900 20 rados_obj.operate() r=0 bl.length=436 2026-03-20T11:22:48.930 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.931+0000 7f6311b5a900 20 zone default found 2026-03-20T11:22:48.930 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.931+0000 7f6311b5a900 4 Realm: () 2026-03-20T11:22:48.930 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.931+0000 7f6311b5a900 4 ZoneGroup: default (217c78dd-3fef-4179-ae38-a328c98d82d1) 2026-03-20T11:22:48.930 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.931+0000 7f6311b5a900 4 Zone: default (7ee099f5-14e5-4754-a80b-09d8b510f566) 2026-03-20T11:22:48.930 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.931+0000 7f6311b5a900 10 cannot find current period zonegroup using local zonegroup configuration 2026-03-20T11:22:48.930 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.931+0000 7f6311b5a900 20 zonegroup default 2026-03-20T11:22:48.930 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.931+0000 7f6311b5a900 20 rados->read ofs=0 len=0 2026-03-20T11:22:48.930 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.931+0000 7f6311b5a900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:22:48.930 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.931+0000 7f6311b5a900 20 rados->read ofs=0 len=0 2026-03-20T11:22:48.930 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.931+0000 7f6311b5a900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:22:48.930 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.931+0000 7f6311b5a900 20 rados->read ofs=0 len=0 2026-03-20T11:22:48.931 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.931+0000 7f6311b5a900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:22:48.931 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.931+0000 7f6311b5a900 20 started sync module instance, tier type = 2026-03-20T11:22:48.931 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.931+0000 7f6311b5a900 20 started zone id=7ee099f5-14e5-4754-a80b-09d8b510f566 (name=default) with tier type = 2026-03-20T11:22:48.933 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.934+0000 7f6311b5a900 20 add_watcher() i=0 2026-03-20T11:22:48.934 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.934+0000 7f6311b5a900 20 add_watcher() i=3 2026-03-20T11:22:48.934 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.934+0000 7f6311b5a900 20 add_watcher() i=5 2026-03-20T11:22:48.934 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.934+0000 7f6311b5a900 20 add_watcher() i=2 2026-03-20T11:22:48.934 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.935+0000 7f6311b5a900 20 add_watcher() i=1 2026-03-20T11:22:48.934 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.935+0000 7f6311b5a900 20 add_watcher() i=7 2026-03-20T11:22:48.934 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.935+0000 7f6311b5a900 20 add_watcher() i=4 2026-03-20T11:22:48.935 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.935+0000 7f6311b5a900 20 add_watcher() i=6 2026-03-20T11:22:48.935 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.935+0000 7f6311b5a900 2 all 8 watchers are set, enabling cache 2026-03-20T11:22:48.936 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.938+0000 7f6311b5a900 20 rgw_check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2026-03-20T11:22:48.936 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.938+0000 7f6311b5a900 20 rgw_check_secure_mon_conn(): mode 1 is insecure 2026-03-20T11:22:48.937 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.938+0000 7f6311b5a900 5 note: GC not initialized 2026-03-20T11:22:48.937 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.938+0000 7f62b6ffd640 20 reqs_thread_entry: start 2026-03-20T11:22:48.971 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.971+0000 7f6311b5a900 20 init_complete bucket index max shards: 11 2026-03-20T11:22:48.971 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.971+0000 7f6311b5a900 20 Filter name: none 2026-03-20T11:22:48.971 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.972+0000 7f62b4ff9640 20 reqs_thread_entry: start 2026-03-20T11:22:48.979 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.980+0000 7f6311b5a900 20 remove_watcher() i=0 2026-03-20T11:22:48.979 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.980+0000 7f6311b5a900 2 removed watcher, disabling cache 2026-03-20T11:22:48.979 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.980+0000 7f6311b5a900 20 remove_watcher() i=1 2026-03-20T11:22:48.979 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.980+0000 7f6311b5a900 20 remove_watcher() i=3 2026-03-20T11:22:48.980 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.981+0000 7f6311b5a900 20 remove_watcher() i=7 2026-03-20T11:22:48.980 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.981+0000 7f6311b5a900 20 remove_watcher() i=2 2026-03-20T11:22:48.980 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.981+0000 7f6311b5a900 20 remove_watcher() i=4 2026-03-20T11:22:48.980 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.981+0000 7f6311b5a900 20 remove_watcher() i=6 2026-03-20T11:22:48.980 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:48.981+0000 7f6311b5a900 20 remove_watcher() i=5 2026-03-20T11:22:48.984 INFO:teuthology.orchestra.run.vm05.stdout:[] 2026-03-20T11:22:48.984 DEBUG:tasks.util.rgw: json result: [] 2026-03-20T11:22:48.984 INFO:tasks.util.rgw:rgwadmin: c1.client.1 : ['zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2026-03-20T11:22:48.984 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.1', '--cluster', 'c1', 'zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2026-03-20T11:22:48.985 DEBUG:teuthology.orchestra.run.vm05:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.1 --cluster c1 zone placement modify --rgw-zone default --placement-id default-placement --compression random 2026-03-20T11:22:49.063 INFO:teuthology.orchestra.run.vm05.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:22:49.063 INFO:teuthology.orchestra.run.vm05.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:22:49.077 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:49.078+0000 7f4cf5331900 20 rgw_check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2026-03-20T11:22:49.077 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:49.078+0000 7f4cf5331900 20 rgw_check_secure_mon_conn(): mode 1 is insecure 2026-03-20T11:22:49.078 INFO:teuthology.orchestra.run.vm05.stderr:2026-03-20T11:22:49.078+0000 7f4c9ffe7640 20 reqs_thread_entry: start 2026-03-20T11:22:49.086 INFO:teuthology.orchestra.run.vm05.stdout:{"id":"7ee099f5-14e5-4754-a80b-09d8b510f566","name":"default","domain_root":"default.rgw.meta:root","control_pool":"default.rgw.control","dedup_pool":"default.rgw.dedup","gc_pool":"default.rgw.log:gc","lc_pool":"default.rgw.log:lc","log_pool":"default.rgw.log","intent_log_pool":"default.rgw.log:intent","usage_log_pool":"default.rgw.log:usage","roles_pool":"default.rgw.meta:roles","reshard_pool":"default.rgw.log:reshard","user_keys_pool":"default.rgw.meta:users.keys","user_email_pool":"default.rgw.meta:users.email","user_swift_pool":"default.rgw.meta:users.swift","user_uid_pool":"default.rgw.meta:users.uid","otp_pool":"default.rgw.otp","notif_pool":"default.rgw.log:notif","topics_pool":"default.rgw.meta:topics","account_pool":"default.rgw.meta:accounts","group_pool":"default.rgw.meta:groups","system_key":{"access_key":"","secret_key":""},"placement_pools":[{"key":"default-placement","val":{"index_pool":"default.rgw.buckets.index","storage_classes":{"STANDARD":{"data_pool":"default.rgw.buckets.data","compression_type":"random"}},"data_extra_pool":"default.rgw.buckets.non-ec","index_type":0,"inline_data":true}}],"realm_id":"","restore_pool":"default.rgw.log:restore"} 2026-03-20T11:22:49.087 DEBUG:tasks.util.rgw: json result: {'id': '7ee099f5-14e5-4754-a80b-09d8b510f566', 'name': 'default', 'domain_root': 'default.rgw.meta:root', 'control_pool': 'default.rgw.control', 'dedup_pool': 'default.rgw.dedup', 'gc_pool': 'default.rgw.log:gc', 'lc_pool': 'default.rgw.log:lc', 'log_pool': 'default.rgw.log', 'intent_log_pool': 'default.rgw.log:intent', 'usage_log_pool': 'default.rgw.log:usage', 'roles_pool': 'default.rgw.meta:roles', 'reshard_pool': 'default.rgw.log:reshard', 'user_keys_pool': 'default.rgw.meta:users.keys', 'user_email_pool': 'default.rgw.meta:users.email', 'user_swift_pool': 'default.rgw.meta:users.swift', 'user_uid_pool': 'default.rgw.meta:users.uid', 'otp_pool': 'default.rgw.otp', 'notif_pool': 'default.rgw.log:notif', 'topics_pool': 'default.rgw.meta:topics', 'account_pool': 'default.rgw.meta:accounts', 'group_pool': 'default.rgw.meta:groups', 'system_key': {'access_key': '', 'secret_key': ''}, 'placement_pools': [{'key': 'default-placement', 'val': {'index_pool': 'default.rgw.buckets.index', 'storage_classes': {'STANDARD': {'data_pool': 'default.rgw.buckets.data', 'compression_type': 'random'}}, 'data_extra_pool': 'default.rgw.buckets.non-ec', 'index_type': 0, 'inline_data': True}}], 'realm_id': '', 'restore_pool': 'default.rgw.log:restore'} 2026-03-20T11:22:49.087 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['user', 'list'] 2026-03-20T11:22:49.087 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'user', 'list'] 2026-03-20T11:22:49.087 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 user list 2026-03-20T11:22:49.126 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:22:49.126 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:22:50.882 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.883+0000 7f852d220900 20 rados->read ofs=0 len=0 2026-03-20T11:22:50.883 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.884+0000 7f852d220900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:22:50.883 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.884+0000 7f852d220900 20 realm 2026-03-20T11:22:50.883 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.884+0000 7f852d220900 20 rados->read ofs=0 len=0 2026-03-20T11:22:50.883 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.884+0000 7f852d220900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:22:50.883 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.884+0000 7f852d220900 4 RGWPeriod::init failed to init realm id : (2) No such file or directory 2026-03-20T11:22:50.883 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.884+0000 7f852d220900 20 rados->read ofs=0 len=0 2026-03-20T11:22:50.883 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.884+0000 7f852d220900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:22:50.883 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.884+0000 7f852d220900 20 rados->read ofs=0 len=0 2026-03-20T11:22:50.884 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.885+0000 7f852d220900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:22:50.884 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.885+0000 7f852d220900 20 rados->read ofs=0 len=0 2026-03-20T11:22:50.884 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.886+0000 7f852d220900 20 rados_obj.operate() r=0 bl.length=1060 2026-03-20T11:22:50.884 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.886+0000 7f852d220900 20 searching for the correct realm 2026-03-20T11:22:50.892 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.893+0000 7f852d220900 20 RGWRados::pool_iterate: got zone_info.8257b8c5-4690-47a7-90b3-f1d84942ce81 2026-03-20T11:22:50.892 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.893+0000 7f852d220900 20 RGWRados::pool_iterate: got default.zonegroup. 2026-03-20T11:22:50.892 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.893+0000 7f852d220900 20 RGWRados::pool_iterate: got default.zone. 2026-03-20T11:22:50.892 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.893+0000 7f852d220900 20 RGWRados::pool_iterate: got zonegroup_info.0a926eb5-09cc-4ba6-8ef1-cde2388e5cbb 2026-03-20T11:22:50.892 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.893+0000 7f852d220900 20 RGWRados::pool_iterate: got zone_names.default 2026-03-20T11:22:50.892 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.893+0000 7f852d220900 20 RGWRados::pool_iterate: got zonegroups_names.default 2026-03-20T11:22:50.892 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.893+0000 7f852d220900 20 rados->read ofs=0 len=0 2026-03-20T11:22:50.892 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.893+0000 7f852d220900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:22:50.892 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.893+0000 7f852d220900 20 rados->read ofs=0 len=0 2026-03-20T11:22:50.892 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.894+0000 7f852d220900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:22:50.892 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.894+0000 7f852d220900 20 rados->read ofs=0 len=0 2026-03-20T11:22:50.893 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.894+0000 7f852d220900 20 rados_obj.operate() r=0 bl.length=436 2026-03-20T11:22:50.893 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.894+0000 7f852d220900 20 zone default found 2026-03-20T11:22:50.893 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.894+0000 7f852d220900 4 Realm: () 2026-03-20T11:22:50.893 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.894+0000 7f852d220900 4 ZoneGroup: default (0a926eb5-09cc-4ba6-8ef1-cde2388e5cbb) 2026-03-20T11:22:50.893 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.894+0000 7f852d220900 4 Zone: default (8257b8c5-4690-47a7-90b3-f1d84942ce81) 2026-03-20T11:22:50.893 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.894+0000 7f852d220900 10 cannot find current period zonegroup using local zonegroup configuration 2026-03-20T11:22:50.893 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.894+0000 7f852d220900 20 zonegroup default 2026-03-20T11:22:50.893 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.894+0000 7f852d220900 20 rados->read ofs=0 len=0 2026-03-20T11:22:50.893 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.894+0000 7f852d220900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:22:50.893 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:50.894+0000 7f852d220900 20 rados->read ofs=0 len=0 2026-03-20T11:22:52.870 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:52.871+0000 7f852d220900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:22:52.870 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:52.871+0000 7f852d220900 20 rados->read ofs=0 len=0 2026-03-20T11:22:52.872 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:52.873+0000 7f852d220900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:22:52.872 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:52.873+0000 7f852d220900 20 started sync module instance, tier type = 2026-03-20T11:22:52.872 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:52.873+0000 7f852d220900 20 started zone id=8257b8c5-4690-47a7-90b3-f1d84942ce81 (name=default) with tier type = 2026-03-20T11:22:54.882 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:54.882+0000 7f852d220900 20 add_watcher() i=0 2026-03-20T11:22:54.882 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:54.883+0000 7f852d220900 20 add_watcher() i=3 2026-03-20T11:22:54.883 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:54.885+0000 7f852d220900 20 add_watcher() i=1 2026-03-20T11:22:54.884 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:54.885+0000 7f852d220900 20 add_watcher() i=2 2026-03-20T11:22:54.884 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:54.885+0000 7f852d220900 20 add_watcher() i=5 2026-03-20T11:22:54.884 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:54.885+0000 7f852d220900 20 add_watcher() i=6 2026-03-20T11:22:54.884 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:54.885+0000 7f852d220900 20 add_watcher() i=7 2026-03-20T11:22:54.884 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:54.885+0000 7f852d220900 20 add_watcher() i=4 2026-03-20T11:22:54.884 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:54.885+0000 7f852d220900 2 all 8 watchers are set, enabling cache 2026-03-20T11:22:54.885 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:54.886+0000 7f8521ffb640 5 boost::asio::awaitable, obj_version> > logback_generations::read(const DoutPrefixProvider*):446: oid=data_loggenerations_metadata not found 2026-03-20T11:22:54.885 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:54.886+0000 7f8521ffb640 20 boost::asio::awaitable {anonymous}::probe_shard(const DoutPrefixProvider*, neorados::RADOS, const neorados::Object&, const neorados::IOContext&, bool&):59 probing obj=data_log.0 2026-03-20T11:22:54.885 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:54.886+0000 7f8521ffb640 20 do_open: entering 2026-03-20T11:22:54.886 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:54.887+0000 7f85217fa640 20 boost::asio::awaitable {anonymous}::probe_shard(const DoutPrefixProvider*, neorados::RADOS, const neorados::Object&, const neorados::IOContext&, bool&):78: obj=data_log.0 does not exist 2026-03-20T11:22:54.886 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:54.887+0000 7f85217fa640 20 boost::asio::awaitable {anonymous}::probe_shard(const DoutPrefixProvider*, neorados::RADOS, const neorados::Object&, const neorados::IOContext&, bool&):59 probing obj=data_log.1 2026-03-20T11:22:54.886 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:54.887+0000 7f85217fa640 20 do_open: entering 2026-03-20T11:22:54.886 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:54.887+0000 7f8520ff9640 20 boost::asio::awaitable {anonymous}::probe_shard(const DoutPrefixProvider*, neorados::RADOS, const neorados::Object&, const neorados::IOContext&, bool&):78: obj=data_log.1 does not exist 2026-03-20T11:22:54.886 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:54.887+0000 7f8520ff9640 20 boost::asio::awaitable {anonymous}::probe_shard(const DoutPrefixProvider*, neorados::RADOS, const neorados::Object&, const neorados::IOContext&, bool&):59 probing obj=data_log.2 2026-03-20T11:22:54.886 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:54.887+0000 7f8520ff9640 20 do_open: entering 2026-03-20T11:22:54.887 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:54.888+0000 7f852997b640 20 boost::asio::awaitable {anonymous}::probe_shard(const DoutPrefixProvider*, neorados::RADOS, const neorados::Object&, const neorados::IOContext&, bool&):78: obj=data_log.2 does not exist 2026-03-20T11:22:54.887 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:54.888+0000 7f852997b640 20 boost::asio::awaitable {anonymous}::probe_shard(const DoutPrefixProvider*, neorados::RADOS, const neorados::Object&, const neorados::IOContext&, bool&):59 probing obj=data_log.3 2026-03-20T11:22:54.887 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:54.888+0000 7f852997b640 20 do_open: entering 2026-03-20T11:22:54.887 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:54.888+0000 7f85237fe640 20 boost::asio::awaitable {anonymous}::probe_shard(const DoutPrefixProvider*, neorados::RADOS, const neorados::Object&, const neorados::IOContext&, bool&):78: obj=data_log.3 does not exist 2026-03-20T11:22:54.887 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:54.888+0000 7f85237fe640 20 do_create: entering 2026-03-20T11:22:54.888 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:54.889+0000 7f8522ffd640 20 do_open: entering 2026-03-20T11:22:54.890 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:54.891+0000 7f852d220900 20 rgw_check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2026-03-20T11:22:54.890 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:54.891+0000 7f852d220900 20 rgw_check_secure_mon_conn(): mode 1 is insecure 2026-03-20T11:22:57.881 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:57.882+0000 7f852d220900 10 rgw_init_ioctx warning: failed to set recovery_priority on default.rgw.meta 2026-03-20T11:22:57.881 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:57.882+0000 7f852d220900 5 note: GC not initialized 2026-03-20T11:22:57.881 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:57.882+0000 7f84d2fe5640 20 reqs_thread_entry: start 2026-03-20T11:22:57.938 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:57.939+0000 7f852d220900 20 init_complete bucket index max shards: 11 2026-03-20T11:22:57.938 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:57.939+0000 7f852d220900 20 Filter name: none 2026-03-20T11:22:57.938 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:57.939+0000 7f84d0fe1640 20 reqs_thread_entry: start 2026-03-20T11:22:57.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:57.948+0000 7f852d220900 20 remove_watcher() i=7 2026-03-20T11:22:57.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:57.949+0000 7f852d220900 2 removed watcher, disabling cache 2026-03-20T11:22:57.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:57.949+0000 7f852d220900 20 remove_watcher() i=5 2026-03-20T11:22:57.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:57.949+0000 7f852d220900 20 remove_watcher() i=6 2026-03-20T11:22:57.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:57.949+0000 7f852d220900 20 remove_watcher() i=4 2026-03-20T11:22:57.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:57.949+0000 7f852d220900 20 remove_watcher() i=0 2026-03-20T11:22:57.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:57.949+0000 7f852d220900 20 remove_watcher() i=1 2026-03-20T11:22:57.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:57.949+0000 7f852d220900 20 remove_watcher() i=3 2026-03-20T11:22:57.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:57.949+0000 7f852d220900 20 remove_watcher() i=2 2026-03-20T11:22:57.954 INFO:teuthology.orchestra.run.vm07.stdout:[] 2026-03-20T11:22:57.954 DEBUG:tasks.util.rgw: json result: [] 2026-03-20T11:22:57.954 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2026-03-20T11:22:57.954 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2026-03-20T11:22:57.954 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 zone placement modify --rgw-zone default --placement-id default-placement --compression random 2026-03-20T11:22:58.033 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:22:58.033 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:22:58.045 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:58.046+0000 7fdf4f922900 20 rgw_check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2026-03-20T11:22:58.045 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:58.046+0000 7fdf4f922900 20 rgw_check_secure_mon_conn(): mode 1 is insecure 2026-03-20T11:22:58.045 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:22:58.046+0000 7fdef97e2640 20 reqs_thread_entry: start 2026-03-20T11:22:58.055 INFO:teuthology.orchestra.run.vm07.stdout:{"id":"8257b8c5-4690-47a7-90b3-f1d84942ce81","name":"default","domain_root":"default.rgw.meta:root","control_pool":"default.rgw.control","dedup_pool":"default.rgw.dedup","gc_pool":"default.rgw.log:gc","lc_pool":"default.rgw.log:lc","log_pool":"default.rgw.log","intent_log_pool":"default.rgw.log:intent","usage_log_pool":"default.rgw.log:usage","roles_pool":"default.rgw.meta:roles","reshard_pool":"default.rgw.log:reshard","user_keys_pool":"default.rgw.meta:users.keys","user_email_pool":"default.rgw.meta:users.email","user_swift_pool":"default.rgw.meta:users.swift","user_uid_pool":"default.rgw.meta:users.uid","otp_pool":"default.rgw.otp","notif_pool":"default.rgw.log:notif","topics_pool":"default.rgw.meta:topics","account_pool":"default.rgw.meta:accounts","group_pool":"default.rgw.meta:groups","system_key":{"access_key":"","secret_key":""},"placement_pools":[{"key":"default-placement","val":{"index_pool":"default.rgw.buckets.index","storage_classes":{"STANDARD":{"data_pool":"default.rgw.buckets.data","compression_type":"random"}},"data_extra_pool":"default.rgw.buckets.non-ec","index_type":0,"inline_data":true}}],"realm_id":"","restore_pool":"default.rgw.log:restore"} 2026-03-20T11:22:58.055 DEBUG:tasks.util.rgw: json result: {'id': '8257b8c5-4690-47a7-90b3-f1d84942ce81', 'name': 'default', 'domain_root': 'default.rgw.meta:root', 'control_pool': 'default.rgw.control', 'dedup_pool': 'default.rgw.dedup', 'gc_pool': 'default.rgw.log:gc', 'lc_pool': 'default.rgw.log:lc', 'log_pool': 'default.rgw.log', 'intent_log_pool': 'default.rgw.log:intent', 'usage_log_pool': 'default.rgw.log:usage', 'roles_pool': 'default.rgw.meta:roles', 'reshard_pool': 'default.rgw.log:reshard', 'user_keys_pool': 'default.rgw.meta:users.keys', 'user_email_pool': 'default.rgw.meta:users.email', 'user_swift_pool': 'default.rgw.meta:users.swift', 'user_uid_pool': 'default.rgw.meta:users.uid', 'otp_pool': 'default.rgw.otp', 'notif_pool': 'default.rgw.log:notif', 'topics_pool': 'default.rgw.meta:topics', 'account_pool': 'default.rgw.meta:accounts', 'group_pool': 'default.rgw.meta:groups', 'system_key': {'access_key': '', 'secret_key': ''}, 'placement_pools': [{'key': 'default-placement', 'val': {'index_pool': 'default.rgw.buckets.index', 'storage_classes': {'STANDARD': {'data_pool': 'default.rgw.buckets.data', 'compression_type': 'random'}}, 'data_extra_pool': 'default.rgw.buckets.non-ec', 'index_type': 0, 'inline_data': True}}], 'realm_id': '', 'restore_pool': 'default.rgw.log:restore'} 2026-03-20T11:22:58.055 INFO:tasks.util.rgw:rgwadmin: c2.client.1 : ['user', 'list'] 2026-03-20T11:22:58.055 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.1', '--cluster', 'c2', 'user', 'list'] 2026-03-20T11:22:58.055 DEBUG:teuthology.orchestra.run.vm08:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.1 --cluster c2 user list 2026-03-20T11:22:58.091 INFO:teuthology.orchestra.run.vm08.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:22:58.091 INFO:teuthology.orchestra.run.vm08.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:22:58.106 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.108+0000 7f9bf0f4f900 20 rados->read ofs=0 len=0 2026-03-20T11:22:58.107 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.109+0000 7f9bf0f4f900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:22:58.107 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.109+0000 7f9bf0f4f900 20 realm 2026-03-20T11:22:58.107 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.109+0000 7f9bf0f4f900 20 rados->read ofs=0 len=0 2026-03-20T11:22:58.107 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.109+0000 7f9bf0f4f900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:22:58.107 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.109+0000 7f9bf0f4f900 4 RGWPeriod::init failed to init realm id : (2) No such file or directory 2026-03-20T11:22:58.107 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.109+0000 7f9bf0f4f900 20 rados->read ofs=0 len=0 2026-03-20T11:22:58.107 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.109+0000 7f9bf0f4f900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:22:58.107 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.109+0000 7f9bf0f4f900 20 rados->read ofs=0 len=0 2026-03-20T11:22:58.108 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.110+0000 7f9bf0f4f900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:22:58.108 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.110+0000 7f9bf0f4f900 20 rados->read ofs=0 len=0 2026-03-20T11:22:58.108 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.110+0000 7f9bf0f4f900 20 rados_obj.operate() r=0 bl.length=1076 2026-03-20T11:22:58.108 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.110+0000 7f9bf0f4f900 20 searching for the correct realm 2026-03-20T11:22:58.116 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.117+0000 7f9bf0f4f900 20 RGWRados::pool_iterate: got zone_info.8257b8c5-4690-47a7-90b3-f1d84942ce81 2026-03-20T11:22:58.116 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.118+0000 7f9bf0f4f900 20 RGWRados::pool_iterate: got default.zonegroup. 2026-03-20T11:22:58.116 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.118+0000 7f9bf0f4f900 20 RGWRados::pool_iterate: got default.zone. 2026-03-20T11:22:58.116 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.118+0000 7f9bf0f4f900 20 RGWRados::pool_iterate: got zonegroup_info.0a926eb5-09cc-4ba6-8ef1-cde2388e5cbb 2026-03-20T11:22:58.116 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.118+0000 7f9bf0f4f900 20 RGWRados::pool_iterate: got zone_names.default 2026-03-20T11:22:58.116 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.118+0000 7f9bf0f4f900 20 RGWRados::pool_iterate: got zonegroups_names.default 2026-03-20T11:22:58.116 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.118+0000 7f9bf0f4f900 20 rados->read ofs=0 len=0 2026-03-20T11:22:58.116 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.118+0000 7f9bf0f4f900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:22:58.116 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.118+0000 7f9bf0f4f900 20 rados->read ofs=0 len=0 2026-03-20T11:22:58.116 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.118+0000 7f9bf0f4f900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:22:58.116 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.118+0000 7f9bf0f4f900 20 rados->read ofs=0 len=0 2026-03-20T11:22:58.116 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.118+0000 7f9bf0f4f900 20 rados_obj.operate() r=0 bl.length=436 2026-03-20T11:22:58.117 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.118+0000 7f9bf0f4f900 20 zone default found 2026-03-20T11:22:58.117 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.118+0000 7f9bf0f4f900 4 Realm: () 2026-03-20T11:22:58.117 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.118+0000 7f9bf0f4f900 4 ZoneGroup: default (0a926eb5-09cc-4ba6-8ef1-cde2388e5cbb) 2026-03-20T11:22:58.117 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.118+0000 7f9bf0f4f900 4 Zone: default (8257b8c5-4690-47a7-90b3-f1d84942ce81) 2026-03-20T11:22:58.117 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.118+0000 7f9bf0f4f900 10 cannot find current period zonegroup using local zonegroup configuration 2026-03-20T11:22:58.117 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.118+0000 7f9bf0f4f900 20 zonegroup default 2026-03-20T11:22:58.117 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.118+0000 7f9bf0f4f900 20 rados->read ofs=0 len=0 2026-03-20T11:22:58.117 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.118+0000 7f9bf0f4f900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:22:58.117 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.118+0000 7f9bf0f4f900 20 rados->read ofs=0 len=0 2026-03-20T11:22:58.117 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.119+0000 7f9bf0f4f900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:22:58.117 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.119+0000 7f9bf0f4f900 20 rados->read ofs=0 len=0 2026-03-20T11:22:58.117 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.119+0000 7f9bf0f4f900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:22:58.117 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.119+0000 7f9bf0f4f900 20 started sync module instance, tier type = 2026-03-20T11:22:58.117 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.119+0000 7f9bf0f4f900 20 started zone id=8257b8c5-4690-47a7-90b3-f1d84942ce81 (name=default) with tier type = 2026-03-20T11:22:58.120 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.122+0000 7f9bf0f4f900 20 add_watcher() i=0 2026-03-20T11:22:58.121 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.122+0000 7f9bf0f4f900 20 add_watcher() i=3 2026-03-20T11:22:58.121 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.122+0000 7f9bf0f4f900 20 add_watcher() i=2 2026-03-20T11:22:58.121 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.123+0000 7f9bf0f4f900 20 add_watcher() i=4 2026-03-20T11:22:58.121 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.123+0000 7f9bf0f4f900 20 add_watcher() i=1 2026-03-20T11:22:58.121 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.123+0000 7f9bf0f4f900 20 add_watcher() i=7 2026-03-20T11:22:58.121 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.123+0000 7f9bf0f4f900 20 add_watcher() i=6 2026-03-20T11:22:58.121 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.123+0000 7f9bf0f4f900 20 add_watcher() i=5 2026-03-20T11:22:58.121 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.123+0000 7f9bf0f4f900 2 all 8 watchers are set, enabling cache 2026-03-20T11:22:58.123 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.125+0000 7f9bf0f4f900 20 rgw_check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2026-03-20T11:22:58.123 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.125+0000 7f9bf0f4f900 20 rgw_check_secure_mon_conn(): mode 1 is insecure 2026-03-20T11:22:58.123 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.125+0000 7f9bf0f4f900 5 note: GC not initialized 2026-03-20T11:22:58.124 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.125+0000 7f9b9a7e4640 20 reqs_thread_entry: start 2026-03-20T11:22:58.160 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.162+0000 7f9bf0f4f900 20 init_complete bucket index max shards: 11 2026-03-20T11:22:58.160 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.162+0000 7f9bf0f4f900 20 Filter name: none 2026-03-20T11:22:58.160 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.162+0000 7f9b4bfff640 20 reqs_thread_entry: start 2026-03-20T11:22:58.170 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.171+0000 7f9bf0f4f900 20 remove_watcher() i=1 2026-03-20T11:22:58.170 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.171+0000 7f9bf0f4f900 2 removed watcher, disabling cache 2026-03-20T11:22:58.170 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.171+0000 7f9bf0f4f900 20 remove_watcher() i=2 2026-03-20T11:22:58.170 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.171+0000 7f9bf0f4f900 20 remove_watcher() i=6 2026-03-20T11:22:58.170 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.171+0000 7f9bf0f4f900 20 remove_watcher() i=0 2026-03-20T11:22:58.170 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.171+0000 7f9bf0f4f900 20 remove_watcher() i=5 2026-03-20T11:22:58.170 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.171+0000 7f9bf0f4f900 20 remove_watcher() i=4 2026-03-20T11:22:58.170 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.171+0000 7f9bf0f4f900 20 remove_watcher() i=3 2026-03-20T11:22:58.170 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.171+0000 7f9bf0f4f900 20 remove_watcher() i=7 2026-03-20T11:22:58.174 INFO:teuthology.orchestra.run.vm08.stdout:[] 2026-03-20T11:22:58.174 DEBUG:tasks.util.rgw: json result: [] 2026-03-20T11:22:58.174 INFO:tasks.util.rgw:rgwadmin: c2.client.1 : ['zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2026-03-20T11:22:58.175 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.1', '--cluster', 'c2', 'zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2026-03-20T11:22:58.175 DEBUG:teuthology.orchestra.run.vm08:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.1 --cluster c2 zone placement modify --rgw-zone default --placement-id default-placement --compression random 2026-03-20T11:22:58.252 INFO:teuthology.orchestra.run.vm08.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:22:58.252 INFO:teuthology.orchestra.run.vm08.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:22:58.265 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.266+0000 7f4e8fe0f900 20 rgw_check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2026-03-20T11:22:58.265 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.266+0000 7f4e8fe0f900 20 rgw_check_secure_mon_conn(): mode 1 is insecure 2026-03-20T11:22:58.265 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-20T11:22:58.267+0000 7f4e397e2640 20 reqs_thread_entry: start 2026-03-20T11:22:58.273 INFO:teuthology.orchestra.run.vm08.stdout:{"id":"8257b8c5-4690-47a7-90b3-f1d84942ce81","name":"default","domain_root":"default.rgw.meta:root","control_pool":"default.rgw.control","dedup_pool":"default.rgw.dedup","gc_pool":"default.rgw.log:gc","lc_pool":"default.rgw.log:lc","log_pool":"default.rgw.log","intent_log_pool":"default.rgw.log:intent","usage_log_pool":"default.rgw.log:usage","roles_pool":"default.rgw.meta:roles","reshard_pool":"default.rgw.log:reshard","user_keys_pool":"default.rgw.meta:users.keys","user_email_pool":"default.rgw.meta:users.email","user_swift_pool":"default.rgw.meta:users.swift","user_uid_pool":"default.rgw.meta:users.uid","otp_pool":"default.rgw.otp","notif_pool":"default.rgw.log:notif","topics_pool":"default.rgw.meta:topics","account_pool":"default.rgw.meta:accounts","group_pool":"default.rgw.meta:groups","system_key":{"access_key":"","secret_key":""},"placement_pools":[{"key":"default-placement","val":{"index_pool":"default.rgw.buckets.index","storage_classes":{"STANDARD":{"data_pool":"default.rgw.buckets.data","compression_type":"random"}},"data_extra_pool":"default.rgw.buckets.non-ec","index_type":0,"inline_data":true}}],"realm_id":"","restore_pool":"default.rgw.log:restore"} 2026-03-20T11:22:58.273 DEBUG:tasks.util.rgw: json result: {'id': '8257b8c5-4690-47a7-90b3-f1d84942ce81', 'name': 'default', 'domain_root': 'default.rgw.meta:root', 'control_pool': 'default.rgw.control', 'dedup_pool': 'default.rgw.dedup', 'gc_pool': 'default.rgw.log:gc', 'lc_pool': 'default.rgw.log:lc', 'log_pool': 'default.rgw.log', 'intent_log_pool': 'default.rgw.log:intent', 'usage_log_pool': 'default.rgw.log:usage', 'roles_pool': 'default.rgw.meta:roles', 'reshard_pool': 'default.rgw.log:reshard', 'user_keys_pool': 'default.rgw.meta:users.keys', 'user_email_pool': 'default.rgw.meta:users.email', 'user_swift_pool': 'default.rgw.meta:users.swift', 'user_uid_pool': 'default.rgw.meta:users.uid', 'otp_pool': 'default.rgw.otp', 'notif_pool': 'default.rgw.log:notif', 'topics_pool': 'default.rgw.meta:topics', 'account_pool': 'default.rgw.meta:accounts', 'group_pool': 'default.rgw.meta:groups', 'system_key': {'access_key': '', 'secret_key': ''}, 'placement_pools': [{'key': 'default-placement', 'val': {'index_pool': 'default.rgw.buckets.index', 'storage_classes': {'STANDARD': {'data_pool': 'default.rgw.buckets.data', 'compression_type': 'random'}}, 'data_extra_pool': 'default.rgw.buckets.non-ec', 'index_type': 0, 'inline_data': True}}], 'realm_id': '', 'restore_pool': 'default.rgw.log:restore'} 2026-03-20T11:22:58.273 INFO:tasks.rgw:Starting rgw... 2026-03-20T11:22:58.273 INFO:tasks.rgw:rgw c1.client.0 config is {'port': 8000} 2026-03-20T11:22:58.273 INFO:tasks.rgw:Using beast as radosgw frontend 2026-03-20T11:22:58.273 DEBUG:teuthology.orchestra.run.vm02:> sudo echo -n http://vm02.local:8000 | sudo tee /home/ubuntu/cephtest/url_file 2026-03-20T11:22:58.298 INFO:teuthology.orchestra.run.vm02.stdout:http://vm02.local:8000 2026-03-20T11:22:58.298 DEBUG:teuthology.orchestra.run.vm02:> sudo chown ceph /home/ubuntu/cephtest/url_file 2026-03-20T11:22:58.358 INFO:tasks.rgw.c1.client.0:Restarting daemon 2026-03-20T11:22:58.358 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8000' -n client.0 --cluster c1 -k /etc/ceph/c1.client.0.keyring --log-file /var/log/ceph/rgw.c1.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c1.client.0.sock --foreground | sudo tee /var/log/ceph/rgw.c1.client.0.stdout 2>&1 2026-03-20T11:22:58.400 INFO:tasks.rgw.c1.client.0:Started 2026-03-20T11:22:58.400 INFO:tasks.rgw:rgw c1.client.1 config is {'port': 8001} 2026-03-20T11:22:58.400 INFO:tasks.rgw:Using beast as radosgw frontend 2026-03-20T11:22:58.400 DEBUG:teuthology.orchestra.run.vm05:> sudo echo -n http://vm05.local:8001 | sudo tee /home/ubuntu/cephtest/url_file 2026-03-20T11:22:58.429 INFO:teuthology.orchestra.run.vm05.stdout:http://vm05.local:8001 2026-03-20T11:22:58.430 DEBUG:teuthology.orchestra.run.vm05:> sudo chown ceph /home/ubuntu/cephtest/url_file 2026-03-20T11:22:58.502 INFO:tasks.rgw.c1.client.1:Restarting daemon 2026-03-20T11:22:58.502 DEBUG:teuthology.orchestra.run.vm05:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8001' -n client.1 --cluster c1 -k /etc/ceph/c1.client.1.keyring --log-file /var/log/ceph/rgw.c1.client.1.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c1.client.1.sock --foreground | sudo tee /var/log/ceph/rgw.c1.client.1.stdout 2>&1 2026-03-20T11:22:58.543 INFO:tasks.rgw.c1.client.1:Started 2026-03-20T11:22:58.543 INFO:tasks.rgw:rgw c2.client.0 config is {'port': 8000} 2026-03-20T11:22:58.544 INFO:tasks.rgw:Using beast as radosgw frontend 2026-03-20T11:22:58.544 DEBUG:teuthology.orchestra.run.vm07:> sudo echo -n http://vm07.local:8000 | sudo tee /home/ubuntu/cephtest/url_file 2026-03-20T11:22:58.574 INFO:teuthology.orchestra.run.vm07.stdout:http://vm07.local:8000 2026-03-20T11:22:58.574 DEBUG:teuthology.orchestra.run.vm07:> sudo chown ceph /home/ubuntu/cephtest/url_file 2026-03-20T11:22:58.643 INFO:tasks.rgw.c2.client.0:Restarting daemon 2026-03-20T11:22:58.643 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8000' -n client.0 --cluster c2 -k /etc/ceph/c2.client.0.keyring --log-file /var/log/ceph/rgw.c2.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c2.client.0.sock --foreground | sudo tee /var/log/ceph/rgw.c2.client.0.stdout 2>&1 2026-03-20T11:22:58.685 INFO:tasks.rgw.c2.client.0:Started 2026-03-20T11:22:58.685 INFO:tasks.rgw:rgw c2.client.1 config is {'port': 8001} 2026-03-20T11:22:58.685 INFO:tasks.rgw:Using beast as radosgw frontend 2026-03-20T11:22:58.685 DEBUG:teuthology.orchestra.run.vm08:> sudo echo -n http://vm08.local:8001 | sudo tee /home/ubuntu/cephtest/url_file 2026-03-20T11:22:58.712 INFO:teuthology.orchestra.run.vm08.stdout:http://vm08.local:8001 2026-03-20T11:22:58.713 DEBUG:teuthology.orchestra.run.vm08:> sudo chown ceph /home/ubuntu/cephtest/url_file 2026-03-20T11:22:58.781 INFO:tasks.rgw.c2.client.1:Restarting daemon 2026-03-20T11:22:58.781 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8001' -n client.1 --cluster c2 -k /etc/ceph/c2.client.1.keyring --log-file /var/log/ceph/rgw.c2.client.1.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c2.client.1.sock --foreground | sudo tee /var/log/ceph/rgw.c2.client.1.stdout 2>&1 2026-03-20T11:22:58.823 INFO:tasks.rgw.c2.client.1:Started 2026-03-20T11:22:58.823 INFO:tasks.rgw:Polling c1.client.0 until it starts accepting connections on http://vm02.local:8000/ 2026-03-20T11:22:58.823 DEBUG:teuthology.orchestra.run.vm02:> curl http://vm02.local:8000/ 2026-03-20T11:22:58.855 INFO:teuthology.orchestra.run.vm02.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2026-03-20T11:22:58.856 INFO:teuthology.orchestra.run.vm02.stderr: Dload Upload Total Spent Left Speed 2026-03-20T11:22:58.858 INFO:teuthology.orchestra.run.vm02.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 187 0 187 0 0 62333 0 --:--:-- --:--:-- --:--:-- 62333 2026-03-20T11:22:58.859 INFO:teuthology.orchestra.run.vm02.stdout:anonymous 2026-03-20T11:22:58.859 INFO:tasks.rgw:Polling c1.client.1 until it starts accepting connections on http://vm05.local:8001/ 2026-03-20T11:22:58.859 DEBUG:teuthology.orchestra.run.vm05:> curl http://vm05.local:8001/ 2026-03-20T11:22:58.896 INFO:teuthology.orchestra.run.vm05.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2026-03-20T11:22:58.896 INFO:teuthology.orchestra.run.vm05.stderr: Dload Upload Total Spent Left Speed 2026-03-20T11:22:58.899 INFO:teuthology.orchestra.run.vm05.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 187 0 187 0 0 62333 0 --:--:-- --:--:-- --:--:-- 93500 2026-03-20T11:22:58.899 INFO:teuthology.orchestra.run.vm05.stdout:anonymous 2026-03-20T11:22:58.899 INFO:tasks.rgw:Polling c2.client.0 until it starts accepting connections on http://vm07.local:8000/ 2026-03-20T11:22:58.900 DEBUG:teuthology.orchestra.run.vm07:> curl http://vm07.local:8000/ 2026-03-20T11:22:58.936 INFO:teuthology.orchestra.run.vm07.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2026-03-20T11:22:58.936 INFO:teuthology.orchestra.run.vm07.stderr: Dload Upload Total Spent Left Speed 2026-03-20T11:22:58.937 INFO:teuthology.orchestra.run.vm07.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 2026-03-20T11:22:58.937 INFO:teuthology.orchestra.run.vm07.stderr:curl: (7) Failed to connect to vm07.local port 8000: Connection refused 2026-03-20T11:22:58.939 DEBUG:teuthology.orchestra.run:got remote process result: 7 2026-03-20T11:22:59.939 DEBUG:teuthology.orchestra.run.vm07:> curl http://vm07.local:8000/ 2026-03-20T11:22:59.959 INFO:teuthology.orchestra.run.vm07.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2026-03-20T11:22:59.959 INFO:teuthology.orchestra.run.vm07.stderr: Dload Upload Total Spent Left Speed 2026-03-20T11:22:59.960 INFO:teuthology.orchestra.run.vm07.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 187 0 187 0 0 182k 0 --:--:-- --:--:-- --:--:-- 182k 2026-03-20T11:22:59.961 INFO:teuthology.orchestra.run.vm07.stdout:anonymous 2026-03-20T11:22:59.961 INFO:tasks.rgw:Polling c2.client.1 until it starts accepting connections on http://vm08.local:8001/ 2026-03-20T11:22:59.961 DEBUG:teuthology.orchestra.run.vm08:> curl http://vm08.local:8001/ 2026-03-20T11:22:59.980 INFO:teuthology.orchestra.run.vm08.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2026-03-20T11:22:59.980 INFO:teuthology.orchestra.run.vm08.stderr: Dload Upload Total Spent Left Speed 2026-03-20T11:22:59.981 INFO:teuthology.orchestra.run.vm08.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 187 0 187 0 0 182k 0 --:--:-- --:--:-- --:--:-- 182k 2026-03-20T11:22:59.982 INFO:teuthology.orchestra.run.vm08.stdout:anonymous 2026-03-20T11:22:59.982 INFO:teuthology.run_tasks:Running task rgw-multisite... 2026-03-20T11:23:00.040 INFO:tasks.rgw_multisite:creating realm.. 2026-03-20T11:23:00.040 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['realm', 'create', '--rgw-realm', 'test-realm', '--default', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:23:00.040 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'realm', 'create', '--rgw-realm', 'test-realm', '--default', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:23:00.040 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 realm create --rgw-realm test-realm --default --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:23:00.076 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:23:00.076 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:23:00.103 INFO:teuthology.orchestra.run.vm02.stdout:{"id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","name":"test-realm","current_period":"5d4b2b15-cd70-4a63-87c8-e59369f9ab35","epoch":1} 2026-03-20T11:23:00.103 INFO:tasks.rgw_multisite:creating master zonegroup.. 2026-03-20T11:23:00.103 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zonegroup', 'set', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--default', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:23:00.103 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zonegroup', 'set', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--default', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:23:00.103 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zonegroup set --rgw-zonegroup a --rgw-realm test-realm --default --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:23:00.176 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:23:00.176 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:23:00.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:23:00.189+0000 7fc75ec12900 -1 old format 2026-03-20T11:23:00.197 INFO:teuthology.orchestra.run.vm02.stdout:{"id":"a","name":"a","api_name":"a","is_master":true,"endpoints":["http://vm02.local:8000"],"hostnames":[],"hostnames_s3website":[],"master_zone":"","zones":[],"placement_targets":[{"name":"default-placement","tags":[],"storage_classes":[]}],"default_placement":"default-placement","realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","sync_policy":{"groups":[]},"enabled_features":["notification_v2","resharding"]} 2026-03-20T11:23:00.197 INFO:tasks.rgw_multisite:creating master zone.. 2026-03-20T11:23:00.197 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zone', 'create', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--default', '--master', '--access-key', 'GRL33KL1ULN5795C', '--secret', 'FPGvuVSJOG1341zbJPfw7ch710f3izIf', '--endpoints', 'http://vm02.local:8000', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:23:00.198 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zone', 'create', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--default', '--master', '--access-key', 'GRL33KL1ULN5795C', '--secret', 'FPGvuVSJOG1341zbJPfw7ch710f3izIf', '--endpoints', 'http://vm02.local:8000', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:23:00.198 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zone create --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --default --master --access-key GRL33KL1ULN5795C --secret FPGvuVSJOG1341zbJPfw7ch710f3izIf --endpoints http://vm02.local:8000 --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:23:00.276 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:23:00.276 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:23:00.288 INFO:teuthology.orchestra.run.vm02.stderr:NOTICE: set zone's realm_id=6a789b89-c2b8-45f5-82cc-fb80234d0c6c 2026-03-20T11:23:00.306 INFO:teuthology.orchestra.run.vm02.stdout:{"id":"4b634ebf-58f5-45bc-839a-cb4b425e773a","name":"a1","domain_root":"a1.rgw.meta:root","control_pool":"a1.rgw.control","dedup_pool":"a1.rgw.dedup","gc_pool":"a1.rgw.log:gc","lc_pool":"a1.rgw.log:lc","log_pool":"a1.rgw.log","intent_log_pool":"a1.rgw.log:intent","usage_log_pool":"a1.rgw.log:usage","roles_pool":"a1.rgw.meta:roles","reshard_pool":"a1.rgw.log:reshard","user_keys_pool":"a1.rgw.meta:users.keys","user_email_pool":"a1.rgw.meta:users.email","user_swift_pool":"a1.rgw.meta:users.swift","user_uid_pool":"a1.rgw.meta:users.uid","otp_pool":"a1.rgw.otp","notif_pool":"a1.rgw.log:notif","topics_pool":"a1.rgw.meta:topics","account_pool":"a1.rgw.meta:accounts","group_pool":"a1.rgw.meta:groups","system_key":{"access_key":"GRL33KL1ULN5795C","secret_key":"FPGvuVSJOG1341zbJPfw7ch710f3izIf"},"placement_pools":[{"key":"default-placement","val":{"index_pool":"a1.rgw.buckets.index","storage_classes":{"STANDARD":{"data_pool":"a1.rgw.buckets.data"}},"data_extra_pool":"a1.rgw.buckets.non-ec","index_type":0,"inline_data":true}}],"realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","restore_pool":"a1.rgw.log:restore"} 2026-03-20T11:23:00.306 DEBUG:teuthology.orchestra.run.vm02:> sudo ceph osd pool create a1.rgw.buckets.data 64 64 --cluster c1 2026-03-20T11:23:00.706 INFO:teuthology.orchestra.run.vm02.stderr:pool 'a1.rgw.buckets.data' created 2026-03-20T11:23:00.728 DEBUG:teuthology.orchestra.run.vm02:> sudo ceph osd pool application enable a1.rgw.buckets.data rgw --cluster c1 2026-03-20T11:23:01.710 INFO:teuthology.orchestra.run.vm02.stderr:enabled application 'rgw' on pool 'a1.rgw.buckets.data' 2026-03-20T11:23:01.745 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zone', 'placement', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', 'modify', '--placement-id', 'default-placement', '--compression', 'random', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:23:01.745 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zone', 'placement', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', 'modify', '--placement-id', 'default-placement', '--compression', 'random', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:23:01.745 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zone placement --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm modify --placement-id default-placement --compression random --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:23:01.786 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:23:01.786 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:23:01.806 INFO:teuthology.orchestra.run.vm02.stdout:{"id":"4b634ebf-58f5-45bc-839a-cb4b425e773a","name":"a1","domain_root":"a1.rgw.meta:root","control_pool":"a1.rgw.control","dedup_pool":"a1.rgw.dedup","gc_pool":"a1.rgw.log:gc","lc_pool":"a1.rgw.log:lc","log_pool":"a1.rgw.log","intent_log_pool":"a1.rgw.log:intent","usage_log_pool":"a1.rgw.log:usage","roles_pool":"a1.rgw.meta:roles","reshard_pool":"a1.rgw.log:reshard","user_keys_pool":"a1.rgw.meta:users.keys","user_email_pool":"a1.rgw.meta:users.email","user_swift_pool":"a1.rgw.meta:users.swift","user_uid_pool":"a1.rgw.meta:users.uid","otp_pool":"a1.rgw.otp","notif_pool":"a1.rgw.log:notif","topics_pool":"a1.rgw.meta:topics","account_pool":"a1.rgw.meta:accounts","group_pool":"a1.rgw.meta:groups","system_key":{"access_key":"GRL33KL1ULN5795C","secret_key":"FPGvuVSJOG1341zbJPfw7ch710f3izIf"},"placement_pools":[{"key":"default-placement","val":{"index_pool":"a1.rgw.buckets.index","storage_classes":{"STANDARD":{"data_pool":"a1.rgw.buckets.data","compression_type":"random"}},"data_extra_pool":"a1.rgw.buckets.non-ec","index_type":0,"inline_data":true}}],"realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","restore_pool":"a1.rgw.log:restore"} 2026-03-20T11:23:01.806 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['period', 'update', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:23:01.806 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'period', 'update', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:23:01.806 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 period update --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --commit --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:23:01.880 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:23:01.880 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:23:01.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:23:01.899+0000 7f07f4130900 0 period (5d4b2b15-cd70-4a63-87c8-e59369f9ab35 does not have zone 4b634ebf-58f5-45bc-839a-cb4b425e773a configured 2026-03-20T11:23:09.155 INFO:teuthology.orchestra.run.vm02.stdout:{"id":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","epoch":1,"predecessor_uuid":"5d4b2b15-cd70-4a63-87c8-e59369f9ab35","sync_status":[],"period_map":{"id":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","zonegroups":[{"id":"a","name":"a","api_name":"a","is_master":true,"endpoints":["http://vm02.local:8000"],"hostnames":[],"hostnames_s3website":[],"master_zone":"4b634ebf-58f5-45bc-839a-cb4b425e773a","zones":[{"id":"4b634ebf-58f5-45bc-839a-cb4b425e773a","name":"a1","endpoints":["http://vm02.local:8000"],"log_meta":false,"log_data":false,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]}],"placement_targets":[{"name":"default-placement","tags":[],"storage_classes":["STANDARD"]}],"default_placement":"default-placement","realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","sync_policy":{"groups":[]},"enabled_features":["notification_v2","resharding"]}],"short_zone_ids":[{"key":"4b634ebf-58f5-45bc-839a-cb4b425e773a","val":162505008}]},"master_zonegroup":"a","master_zone":"4b634ebf-58f5-45bc-839a-cb4b425e773a","period_config":{"bucket_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false},"bucket_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false},"anonymous_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false}},"realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","realm_epoch":2} 2026-03-20T11:23:09.155 DEBUG:tasks.rgw.c1.client.0:waiting for process to exit 2026-03-20T11:23:09.155 INFO:teuthology.orchestra.run:waiting for 300 2026-03-20T11:23:09.196 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:23:09.197+0000 7f5796afb640 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper term radosgw --rgw-frontends beast port=8000 -n client.0 --cluster c1 -k /etc/ceph/c1.client.0.keyring --log-file /var/log/ceph/rgw.c1.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c1.client.0.sock --foreground (PID: 61543) UID: 0 2026-03-20T11:23:09.196 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:23:09.197+0000 7f579a594980 -1 shutting down 2026-03-20T11:23:31.565 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.0 is failed for ~0s 2026-03-20T11:23:33.560 INFO:tasks.rgw.c1.client.0:Stopped 2026-03-20T11:23:33.560 INFO:tasks.rgw.c1.client.0:Restarting daemon 2026-03-20T11:23:33.560 DEBUG:teuthology.orchestra.run.vm02:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8000' -n client.0 --cluster c1 -k /etc/ceph/c1.client.0.keyring --log-file /var/log/ceph/rgw.c1.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c1.client.0.sock --foreground --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm | sudo tee /var/log/ceph/rgw.c1.client.0.stdout 2>&1 2026-03-20T11:23:33.562 INFO:tasks.rgw.c1.client.0:Started 2026-03-20T11:23:33.562 DEBUG:teuthology.orchestra.run.vm02:> curl http://vm02.local:8000 2026-03-20T11:23:33.582 INFO:teuthology.orchestra.run.vm02.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2026-03-20T11:23:33.582 INFO:teuthology.orchestra.run.vm02.stderr: Dload Upload Total Spent Left Speed 2026-03-20T11:23:33.582 INFO:teuthology.orchestra.run.vm02.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 2026-03-20T11:23:33.582 INFO:teuthology.orchestra.run.vm02.stderr:curl: (7) Failed to connect to vm02.local port 8000: Connection refused 2026-03-20T11:23:33.583 DEBUG:teuthology.orchestra.run:got remote process result: 7 2026-03-20T11:23:34.583 DEBUG:teuthology.orchestra.run.vm02:> curl http://vm02.local:8000 2026-03-20T11:23:34.602 INFO:teuthology.orchestra.run.vm02.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2026-03-20T11:23:34.602 INFO:teuthology.orchestra.run.vm02.stderr: Dload Upload Total Spent Left Speed 2026-03-20T11:23:34.604 INFO:teuthology.orchestra.run.vm02.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 187 0 187 0 0 182k 0 --:--:-- --:--:-- --:--:-- 182k 2026-03-20T11:23:34.604 INFO:teuthology.orchestra.run.vm02.stdout:anonymous 2026-03-20T11:23:34.604 INFO:tasks.rgw_multisite:creating admin user.. 2026-03-20T11:23:34.604 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['user', 'create', '--uid', 'realm-admin', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--display-name', 'Realm Admin', '--system', '--access-key', 'GRL33KL1ULN5795C', '--secret', 'FPGvuVSJOG1341zbJPfw7ch710f3izIf', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:23:34.604 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'user', 'create', '--uid', 'realm-admin', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--display-name', 'Realm Admin', '--system', '--access-key', 'GRL33KL1ULN5795C', '--secret', 'FPGvuVSJOG1341zbJPfw7ch710f3izIf', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:23:34.604 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 user create --uid realm-admin --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --display-name 'Realm Admin' --system --access-key GRL33KL1ULN5795C --secret FPGvuVSJOG1341zbJPfw7ch710f3izIf --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:23:34.685 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:23:34.685 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:23:34.752 INFO:teuthology.orchestra.run.vm02.stdout:{"user_id":"realm-admin","display_name":"Realm Admin","email":"","suspended":0,"max_buckets":1000,"subusers":[],"keys":[{"user":"realm-admin","access_key":"GRL33KL1ULN5795C","secret_key":"FPGvuVSJOG1341zbJPfw7ch710f3izIf","active":true,"create_date":"2026-03-20T11:23:34.751454Z"}],"swift_keys":[],"caps":[],"op_mask":"read, write, delete","system":true,"default_placement":"","default_storage_class":"","placement_tags":[],"bucket_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"temp_url_keys":[],"type":"rgw","mfa_ids":[],"account_id":"","path":"/","create_date":"2026-03-20T11:23:34.751447Z","tags":[],"group_ids":[]} 2026-03-20T11:23:34.761 INFO:tasks.rgw_multisite:creating zone.. 2026-03-20T11:23:34.761 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zone', 'create', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--access-key', 'GRL33KL1ULN5795C', '--secret', 'FPGvuVSJOG1341zbJPfw7ch710f3izIf', '--endpoints', 'http://vm05.local:8001', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:23:34.761 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zone', 'create', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--access-key', 'GRL33KL1ULN5795C', '--secret', 'FPGvuVSJOG1341zbJPfw7ch710f3izIf', '--endpoints', 'http://vm05.local:8001', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:23:34.761 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zone create --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --access-key GRL33KL1ULN5795C --secret FPGvuVSJOG1341zbJPfw7ch710f3izIf --endpoints http://vm05.local:8001 --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:23:34.840 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:23:34.840 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:23:34.854 INFO:teuthology.orchestra.run.vm02.stderr:NOTICE: set zone's realm_id=6a789b89-c2b8-45f5-82cc-fb80234d0c6c 2026-03-20T11:23:34.872 INFO:teuthology.orchestra.run.vm02.stdout:{"id":"51055576-6662-409d-8abb-ecaeefd3dd80","name":"a2","domain_root":"a2.rgw.meta:root","control_pool":"a2.rgw.control","dedup_pool":"a2.rgw.dedup","gc_pool":"a2.rgw.log:gc","lc_pool":"a2.rgw.log:lc","log_pool":"a2.rgw.log","intent_log_pool":"a2.rgw.log:intent","usage_log_pool":"a2.rgw.log:usage","roles_pool":"a2.rgw.meta:roles","reshard_pool":"a2.rgw.log:reshard","user_keys_pool":"a2.rgw.meta:users.keys","user_email_pool":"a2.rgw.meta:users.email","user_swift_pool":"a2.rgw.meta:users.swift","user_uid_pool":"a2.rgw.meta:users.uid","otp_pool":"a2.rgw.otp","notif_pool":"a2.rgw.log:notif","topics_pool":"a2.rgw.meta:topics","account_pool":"a2.rgw.meta:accounts","group_pool":"a2.rgw.meta:groups","system_key":{"access_key":"GRL33KL1ULN5795C","secret_key":"FPGvuVSJOG1341zbJPfw7ch710f3izIf"},"placement_pools":[{"key":"default-placement","val":{"index_pool":"a2.rgw.buckets.index","storage_classes":{"STANDARD":{"data_pool":"a2.rgw.buckets.data"}},"data_extra_pool":"a2.rgw.buckets.non-ec","index_type":0,"inline_data":true}}],"realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","restore_pool":"a2.rgw.log:restore"} 2026-03-20T11:23:34.872 DEBUG:teuthology.orchestra.run.vm05:> sudo ceph osd pool create a2.rgw.buckets.data 64 64 --cluster c1 2026-03-20T11:23:35.077 INFO:teuthology.orchestra.run.vm05.stderr:pool 'a2.rgw.buckets.data' created 2026-03-20T11:23:35.104 DEBUG:teuthology.orchestra.run.vm05:> sudo ceph osd pool application enable a2.rgw.buckets.data rgw --cluster c1 2026-03-20T11:23:36.081 INFO:teuthology.orchestra.run.vm05.stderr:enabled application 'rgw' on pool 'a2.rgw.buckets.data' 2026-03-20T11:23:36.109 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zone', 'placement', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', 'modify', '--placement-id', 'default-placement', '--compression', 'random', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:23:36.109 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zone', 'placement', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', 'modify', '--placement-id', 'default-placement', '--compression', 'random', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:23:36.109 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zone placement --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm modify --placement-id default-placement --compression random --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:23:36.151 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:23:36.151 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:23:36.172 INFO:teuthology.orchestra.run.vm02.stdout:{"id":"51055576-6662-409d-8abb-ecaeefd3dd80","name":"a2","domain_root":"a2.rgw.meta:root","control_pool":"a2.rgw.control","dedup_pool":"a2.rgw.dedup","gc_pool":"a2.rgw.log:gc","lc_pool":"a2.rgw.log:lc","log_pool":"a2.rgw.log","intent_log_pool":"a2.rgw.log:intent","usage_log_pool":"a2.rgw.log:usage","roles_pool":"a2.rgw.meta:roles","reshard_pool":"a2.rgw.log:reshard","user_keys_pool":"a2.rgw.meta:users.keys","user_email_pool":"a2.rgw.meta:users.email","user_swift_pool":"a2.rgw.meta:users.swift","user_uid_pool":"a2.rgw.meta:users.uid","otp_pool":"a2.rgw.otp","notif_pool":"a2.rgw.log:notif","topics_pool":"a2.rgw.meta:topics","account_pool":"a2.rgw.meta:accounts","group_pool":"a2.rgw.meta:groups","system_key":{"access_key":"GRL33KL1ULN5795C","secret_key":"FPGvuVSJOG1341zbJPfw7ch710f3izIf"},"placement_pools":[{"key":"default-placement","val":{"index_pool":"a2.rgw.buckets.index","storage_classes":{"STANDARD":{"data_pool":"a2.rgw.buckets.data","compression_type":"random"}},"data_extra_pool":"a2.rgw.buckets.non-ec","index_type":0,"inline_data":true}}],"realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","restore_pool":"a2.rgw.log:restore"} 2026-03-20T11:23:36.173 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['period', 'update', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:23:36.173 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'period', 'update', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:23:36.173 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 period update --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --commit --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:23:36.246 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:23:36.246 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:23:36.262 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:23:36.264+0000 7fe814d26900 0 period (cb2fcdce-2c1b-429a-8601-a284ee3a2381 does not have zone 51055576-6662-409d-8abb-ecaeefd3dd80 configured 2026-03-20T11:23:37.571 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.0 has been restored 2026-03-20T11:23:43.185 INFO:teuthology.orchestra.run.vm02.stderr:Sending period to new master zone 4b634ebf-58f5-45bc-839a-cb4b425e773a 2026-03-20T11:23:43.211 INFO:teuthology.orchestra.run.vm02.stdout:{"id":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","epoch":2,"predecessor_uuid":"5d4b2b15-cd70-4a63-87c8-e59369f9ab35","sync_status":[],"period_map":{"id":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","zonegroups":[{"id":"a","name":"a","api_name":"a","is_master":true,"endpoints":["http://vm02.local:8000"],"hostnames":[],"hostnames_s3website":[],"master_zone":"4b634ebf-58f5-45bc-839a-cb4b425e773a","zones":[{"id":"4b634ebf-58f5-45bc-839a-cb4b425e773a","name":"a1","endpoints":["http://vm02.local:8000"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]},{"id":"51055576-6662-409d-8abb-ecaeefd3dd80","name":"a2","endpoints":["http://vm05.local:8001"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]}],"placement_targets":[{"name":"default-placement","tags":[],"storage_classes":["STANDARD"]}],"default_placement":"default-placement","realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","sync_policy":{"groups":[]},"enabled_features":["notification_v2","resharding"]}],"short_zone_ids":[{"key":"4b634ebf-58f5-45bc-839a-cb4b425e773a","val":162505008},{"key":"51055576-6662-409d-8abb-ecaeefd3dd80","val":2776933003}]},"master_zonegroup":"a","master_zone":"4b634ebf-58f5-45bc-839a-cb4b425e773a","period_config":{"bucket_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false},"bucket_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false},"anonymous_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false}},"realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","realm_epoch":2} 2026-03-20T11:23:43.211 DEBUG:tasks.rgw.c1.client.1:waiting for process to exit 2026-03-20T11:23:43.211 INFO:teuthology.orchestra.run:waiting for 300 2026-03-20T11:23:43.212 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:23:43.213+0000 7f1961e75640 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper term radosgw --rgw-frontends beast port=8001 -n client.1 --cluster c1 -k /etc/ceph/c1.client.1.keyring --log-file /var/log/ceph/rgw.c1.client.1.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c1.client.1.sock --foreground (PID: 56796) UID: 0 2026-03-20T11:23:43.212 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:23:43.213+0000 7f1965917980 -1 shutting down 2026-03-20T11:24:01.490 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~0s 2026-03-20T11:24:01.515 INFO:tasks.rgw.c1.client.1:Stopped 2026-03-20T11:24:01.515 INFO:tasks.rgw.c1.client.1:Restarting daemon 2026-03-20T11:24:01.515 DEBUG:teuthology.orchestra.run.vm05:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8001' -n client.1 --cluster c1 -k /etc/ceph/c1.client.1.keyring --log-file /var/log/ceph/rgw.c1.client.1.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c1.client.1.sock --foreground --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm | sudo tee /var/log/ceph/rgw.c1.client.1.stdout 2>&1 2026-03-20T11:24:01.516 INFO:tasks.rgw.c1.client.1:Started 2026-03-20T11:24:01.516 DEBUG:teuthology.orchestra.run.vm05:> curl http://vm05.local:8001 2026-03-20T11:24:01.535 DEBUG:teuthology.orchestra.run:got remote process result: 7 2026-03-20T11:24:01.535 INFO:teuthology.orchestra.run.vm05.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2026-03-20T11:24:01.535 INFO:teuthology.orchestra.run.vm05.stderr: Dload Upload Total Spent Left Speed 2026-03-20T11:24:01.535 INFO:teuthology.orchestra.run.vm05.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 2026-03-20T11:24:01.535 INFO:teuthology.orchestra.run.vm05.stderr:curl: (7) Failed to connect to vm05.local port 8001: Connection refused 2026-03-20T11:24:02.536 DEBUG:teuthology.orchestra.run.vm05:> curl http://vm05.local:8001 2026-03-20T11:24:02.555 INFO:teuthology.orchestra.run.vm05.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2026-03-20T11:24:02.555 INFO:teuthology.orchestra.run.vm05.stderr: Dload Upload Total Spent Left Speed 2026-03-20T11:24:02.556 INFO:teuthology.orchestra.run.vm05.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 187 0 187 0 0 182k 0 --:--:-- --:--:-- --:--:-- 182k 2026-03-20T11:24:02.556 INFO:teuthology.orchestra.run.vm05.stdout:anonymous 2026-03-20T11:24:02.556 INFO:tasks.rgw_multisite:pulling realm configuration to c2 2026-03-20T11:24:02.556 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['realm', 'pull', '--rgw-realm', 'test-realm', '--default', '--url', 'http://vm02.local:8000', '--access-key', 'GRL33KL1ULN5795C', '--secret', 'FPGvuVSJOG1341zbJPfw7ch710f3izIf', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:24:02.556 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'realm', 'pull', '--rgw-realm', 'test-realm', '--default', '--url', 'http://vm02.local:8000', '--access-key', 'GRL33KL1ULN5795C', '--secret', 'FPGvuVSJOG1341zbJPfw7ch710f3izIf', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:24:02.556 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 realm pull --rgw-realm test-realm --default --url http://vm02.local:8000 --access-key GRL33KL1ULN5795C --secret FPGvuVSJOG1341zbJPfw7ch710f3izIf --cluster c2 --debug-rgw 0 --debug-ms 0 2026-03-20T11:24:02.595 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:24:02.595 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:24:04.360 INFO:teuthology.orchestra.run.vm07.stdout:{"id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","name":"test-realm","current_period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","epoch":2} 2026-03-20T11:24:04.360 INFO:tasks.rgw_multisite:creating zonegroup.. 2026-03-20T11:24:04.360 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['zonegroup', 'set', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--default', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:24:04.360 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'zonegroup', 'set', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--default', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:24:04.361 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 zonegroup set --rgw-zonegroup b --rgw-realm test-realm --default --cluster c2 --debug-rgw 0 --debug-ms 0 2026-03-20T11:24:04.440 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:24:04.440 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:24:04.453 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:24:04.456+0000 7fb856e10900 -1 old format 2026-03-20T11:24:04.462 INFO:teuthology.orchestra.run.vm07.stdout:{"id":"b","name":"b","api_name":"b","is_master":false,"endpoints":["http://vm07.local:8000"],"hostnames":[],"hostnames_s3website":[],"master_zone":"","zones":[],"placement_targets":[{"name":"default-placement","tags":[],"storage_classes":[]}],"default_placement":"default-placement","realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","sync_policy":{"groups":[]},"enabled_features":["notification_v2","resharding"]} 2026-03-20T11:24:04.462 INFO:tasks.rgw_multisite:creating zone.. 2026-03-20T11:24:04.462 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['zone', 'create', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--default', '--master', '--access-key', 'GRL33KL1ULN5795C', '--secret', 'FPGvuVSJOG1341zbJPfw7ch710f3izIf', '--endpoints', 'http://vm07.local:8000', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:24:04.462 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'zone', 'create', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--default', '--master', '--access-key', 'GRL33KL1ULN5795C', '--secret', 'FPGvuVSJOG1341zbJPfw7ch710f3izIf', '--endpoints', 'http://vm07.local:8000', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:24:04.463 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 zone create --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --default --master --access-key GRL33KL1ULN5795C --secret FPGvuVSJOG1341zbJPfw7ch710f3izIf --endpoints http://vm07.local:8000 --cluster c2 --debug-rgw 0 --debug-ms 0 2026-03-20T11:24:04.545 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:24:04.545 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:24:04.560 INFO:teuthology.orchestra.run.vm07.stderr:NOTICE: set zone's realm_id=6a789b89-c2b8-45f5-82cc-fb80234d0c6c 2026-03-20T11:24:04.581 INFO:teuthology.orchestra.run.vm07.stdout:{"id":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","name":"b1","domain_root":"b1.rgw.meta:root","control_pool":"b1.rgw.control","dedup_pool":"b1.rgw.dedup","gc_pool":"b1.rgw.log:gc","lc_pool":"b1.rgw.log:lc","log_pool":"b1.rgw.log","intent_log_pool":"b1.rgw.log:intent","usage_log_pool":"b1.rgw.log:usage","roles_pool":"b1.rgw.meta:roles","reshard_pool":"b1.rgw.log:reshard","user_keys_pool":"b1.rgw.meta:users.keys","user_email_pool":"b1.rgw.meta:users.email","user_swift_pool":"b1.rgw.meta:users.swift","user_uid_pool":"b1.rgw.meta:users.uid","otp_pool":"b1.rgw.otp","notif_pool":"b1.rgw.log:notif","topics_pool":"b1.rgw.meta:topics","account_pool":"b1.rgw.meta:accounts","group_pool":"b1.rgw.meta:groups","system_key":{"access_key":"GRL33KL1ULN5795C","secret_key":"FPGvuVSJOG1341zbJPfw7ch710f3izIf"},"placement_pools":[{"key":"default-placement","val":{"index_pool":"b1.rgw.buckets.index","storage_classes":{"STANDARD":{"data_pool":"b1.rgw.buckets.data"}},"data_extra_pool":"b1.rgw.buckets.non-ec","index_type":0,"inline_data":true}}],"realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","restore_pool":"b1.rgw.log:restore"} 2026-03-20T11:24:04.581 DEBUG:teuthology.orchestra.run.vm07:> sudo ceph osd pool create b1.rgw.buckets.data 64 64 --cluster c2 2026-03-20T11:24:04.873 INFO:teuthology.orchestra.run.vm07.stderr:pool 'b1.rgw.buckets.data' created 2026-03-20T11:24:04.904 DEBUG:teuthology.orchestra.run.vm07:> sudo ceph osd pool application enable b1.rgw.buckets.data rgw --cluster c2 2026-03-20T11:24:05.877 INFO:teuthology.orchestra.run.vm07.stderr:enabled application 'rgw' on pool 'b1.rgw.buckets.data' 2026-03-20T11:24:05.910 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['zone', 'placement', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', 'modify', '--placement-id', 'default-placement', '--compression', 'random', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:24:05.910 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'zone', 'placement', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', 'modify', '--placement-id', 'default-placement', '--compression', 'random', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:24:05.910 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 zone placement --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm modify --placement-id default-placement --compression random --cluster c2 --debug-rgw 0 --debug-ms 0 2026-03-20T11:24:05.949 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:24:05.949 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:24:05.968 INFO:teuthology.orchestra.run.vm07.stdout:{"id":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","name":"b1","domain_root":"b1.rgw.meta:root","control_pool":"b1.rgw.control","dedup_pool":"b1.rgw.dedup","gc_pool":"b1.rgw.log:gc","lc_pool":"b1.rgw.log:lc","log_pool":"b1.rgw.log","intent_log_pool":"b1.rgw.log:intent","usage_log_pool":"b1.rgw.log:usage","roles_pool":"b1.rgw.meta:roles","reshard_pool":"b1.rgw.log:reshard","user_keys_pool":"b1.rgw.meta:users.keys","user_email_pool":"b1.rgw.meta:users.email","user_swift_pool":"b1.rgw.meta:users.swift","user_uid_pool":"b1.rgw.meta:users.uid","otp_pool":"b1.rgw.otp","notif_pool":"b1.rgw.log:notif","topics_pool":"b1.rgw.meta:topics","account_pool":"b1.rgw.meta:accounts","group_pool":"b1.rgw.meta:groups","system_key":{"access_key":"GRL33KL1ULN5795C","secret_key":"FPGvuVSJOG1341zbJPfw7ch710f3izIf"},"placement_pools":[{"key":"default-placement","val":{"index_pool":"b1.rgw.buckets.index","storage_classes":{"STANDARD":{"data_pool":"b1.rgw.buckets.data","compression_type":"random"}},"data_extra_pool":"b1.rgw.buckets.non-ec","index_type":0,"inline_data":true}}],"realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","restore_pool":"b1.rgw.log:restore"} 2026-03-20T11:24:05.968 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['period', 'update', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:24:05.968 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'period', 'update', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:24:05.968 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 period update --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --commit --cluster c2 --debug-rgw 0 --debug-ms 0 2026-03-20T11:24:06.043 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:24:06.043 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:24:06.059 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:24:06.062+0000 7feb5972c900 0 period (cb2fcdce-2c1b-429a-8601-a284ee3a2381 does not have zone efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c configured 2026-03-20T11:24:07.496 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 has been restored 2026-03-20T11:24:12.975 INFO:teuthology.orchestra.run.vm07.stderr:Sending period to new master zone 4b634ebf-58f5-45bc-839a-cb4b425e773a 2026-03-20T11:24:13.010 INFO:teuthology.orchestra.run.vm07.stdout:{"id":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","epoch":3,"predecessor_uuid":"5d4b2b15-cd70-4a63-87c8-e59369f9ab35","sync_status":[],"period_map":{"id":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","zonegroups":[{"id":"a","name":"a","api_name":"a","is_master":true,"endpoints":["http://vm02.local:8000"],"hostnames":[],"hostnames_s3website":[],"master_zone":"4b634ebf-58f5-45bc-839a-cb4b425e773a","zones":[{"id":"4b634ebf-58f5-45bc-839a-cb4b425e773a","name":"a1","endpoints":["http://vm02.local:8000"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]},{"id":"51055576-6662-409d-8abb-ecaeefd3dd80","name":"a2","endpoints":["http://vm05.local:8001"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]}],"placement_targets":[{"name":"default-placement","tags":[],"storage_classes":["STANDARD"]}],"default_placement":"default-placement","realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","sync_policy":{"groups":[]},"enabled_features":["notification_v2","resharding"]},{"id":"b","name":"b","api_name":"b","is_master":false,"endpoints":["http://vm07.local:8000"],"hostnames":[],"hostnames_s3website":[],"master_zone":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","zones":[{"id":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","name":"b1","endpoints":["http://vm07.local:8000"],"log_meta":false,"log_data":false,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]}],"placement_targets":[{"name":"default-placement","tags":[],"storage_classes":["STANDARD"]}],"default_placement":"default-placement","realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","sync_policy":{"groups":[]},"enabled_features":["notification_v2","resharding"]}],"short_zone_ids":[{"key":"4b634ebf-58f5-45bc-839a-cb4b425e773a","val":162505008},{"key":"51055576-6662-409d-8abb-ecaeefd3dd80","val":2776933003},{"key":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","val":114662526}]},"master_zonegroup":"a","master_zone":"4b634ebf-58f5-45bc-839a-cb4b425e773a","period_config":{"bucket_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false},"bucket_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false},"anonymous_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false}},"realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","realm_epoch":2} 2026-03-20T11:24:13.011 DEBUG:tasks.rgw.c2.client.0:waiting for process to exit 2026-03-20T11:24:13.011 INFO:teuthology.orchestra.run:waiting for 300 2026-03-20T11:24:13.051 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:24:13.054+0000 7fdc0b91d640 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper term radosgw --rgw-frontends beast port=8000 -n client.0 --cluster c2 -k /etc/ceph/c2.client.0.keyring --log-file /var/log/ceph/rgw.c2.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c2.client.0.sock --foreground (PID: 58552) UID: 0 2026-03-20T11:24:13.051 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:24:13.054+0000 7fdc11325980 -1 shutting down 2026-03-20T11:24:31.314 INFO:tasks.rgw.c2.client.0:Stopped 2026-03-20T11:24:31.314 INFO:tasks.rgw.c2.client.0:Restarting daemon 2026-03-20T11:24:31.314 DEBUG:teuthology.orchestra.run.vm07:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8000' -n client.0 --cluster c2 -k /etc/ceph/c2.client.0.keyring --log-file /var/log/ceph/rgw.c2.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c2.client.0.sock --foreground --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm | sudo tee /var/log/ceph/rgw.c2.client.0.stdout 2>&1 2026-03-20T11:24:31.316 INFO:tasks.rgw.c2.client.0:Started 2026-03-20T11:24:31.316 DEBUG:teuthology.orchestra.run.vm07:> curl http://vm07.local:8000 2026-03-20T11:24:31.347 DEBUG:teuthology.orchestra.run:got remote process result: 7 2026-03-20T11:24:31.347 INFO:teuthology.orchestra.run.vm07.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2026-03-20T11:24:31.347 INFO:teuthology.orchestra.run.vm07.stderr: Dload Upload Total Spent Left Speed 2026-03-20T11:24:31.347 INFO:teuthology.orchestra.run.vm07.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 2026-03-20T11:24:31.347 INFO:teuthology.orchestra.run.vm07.stderr:curl: (7) Failed to connect to vm07.local port 8000: Connection refused 2026-03-20T11:24:32.347 DEBUG:teuthology.orchestra.run.vm07:> curl http://vm07.local:8000 2026-03-20T11:24:32.371 INFO:teuthology.orchestra.run.vm07.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2026-03-20T11:24:32.371 INFO:teuthology.orchestra.run.vm07.stderr: Dload Upload Total Spent Left Speed 2026-03-20T11:24:32.371 INFO:teuthology.orchestra.run.vm07.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 2026-03-20T11:24:32.372 INFO:teuthology.orchestra.run.vm07.stderr:curl: (7) Failed to connect to vm07.local port 8000: Connection refused 2026-03-20T11:24:32.372 DEBUG:teuthology.orchestra.run:got remote process result: 7 2026-03-20T11:24:34.373 DEBUG:teuthology.orchestra.run.vm07:> curl http://vm07.local:8000 2026-03-20T11:24:34.395 INFO:teuthology.orchestra.run.vm07.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2026-03-20T11:24:34.395 INFO:teuthology.orchestra.run.vm07.stderr: Dload Upload Total Spent Left Speed 2026-03-20T11:24:34.395 INFO:teuthology.orchestra.run.vm07.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 2026-03-20T11:24:34.395 INFO:teuthology.orchestra.run.vm07.stderr:curl: (7) Failed to connect to vm07.local port 8000: Connection refused 2026-03-20T11:24:34.395 DEBUG:teuthology.orchestra.run:got remote process result: 7 2026-03-20T11:24:38.396 DEBUG:teuthology.orchestra.run.vm07:> curl http://vm07.local:8000 2026-03-20T11:24:38.415 INFO:teuthology.orchestra.run.vm07.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2026-03-20T11:24:38.415 INFO:teuthology.orchestra.run.vm07.stderr: Dload Upload Total Spent Left Speed 2026-03-20T11:24:38.417 INFO:teuthology.orchestra.run.vm07.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 187 0 187 0 0 93500 0 --:--:-- --:--:-- --:--:-- 93500 2026-03-20T11:24:38.417 INFO:teuthology.orchestra.run.vm07.stdout:anonymous 2026-03-20T11:24:38.418 INFO:tasks.rgw_multisite:pulling realm configuration to c2 2026-03-20T11:24:38.418 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['realm', 'pull', '--rgw-realm', 'test-realm', '--default', '--url', 'http://vm02.local:8000', '--access-key', 'GRL33KL1ULN5795C', '--secret', 'FPGvuVSJOG1341zbJPfw7ch710f3izIf', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:24:38.418 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'realm', 'pull', '--rgw-realm', 'test-realm', '--default', '--url', 'http://vm02.local:8000', '--access-key', 'GRL33KL1ULN5795C', '--secret', 'FPGvuVSJOG1341zbJPfw7ch710f3izIf', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:24:38.418 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 realm pull --rgw-realm test-realm --default --url http://vm02.local:8000 --access-key GRL33KL1ULN5795C --secret FPGvuVSJOG1341zbJPfw7ch710f3izIf --cluster c2 --debug-rgw 0 --debug-ms 0 2026-03-20T11:24:38.498 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:24:38.498 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:24:38.540 INFO:teuthology.orchestra.run.vm07.stdout:{"id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","name":"test-realm","current_period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","epoch":2} 2026-03-20T11:24:38.540 INFO:tasks.rgw_multisite:creating zone.. 2026-03-20T11:24:38.540 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['zone', 'create', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--access-key', 'GRL33KL1ULN5795C', '--secret', 'FPGvuVSJOG1341zbJPfw7ch710f3izIf', '--endpoints', 'http://vm08.local:8001', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:24:38.540 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'zone', 'create', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--access-key', 'GRL33KL1ULN5795C', '--secret', 'FPGvuVSJOG1341zbJPfw7ch710f3izIf', '--endpoints', 'http://vm08.local:8001', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:24:38.540 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 zone create --rgw-zone b2 --rgw-zonegroup b --rgw-realm test-realm --access-key GRL33KL1ULN5795C --secret FPGvuVSJOG1341zbJPfw7ch710f3izIf --endpoints http://vm08.local:8001 --cluster c2 --debug-rgw 0 --debug-ms 0 2026-03-20T11:24:38.625 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:24:38.625 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:24:38.642 INFO:teuthology.orchestra.run.vm07.stderr:NOTICE: set zone's realm_id=6a789b89-c2b8-45f5-82cc-fb80234d0c6c 2026-03-20T11:24:38.664 INFO:teuthology.orchestra.run.vm07.stdout:{"id":"cd52b229-0d4a-43bf-b4ba-156aa19e7aef","name":"b2","domain_root":"b2.rgw.meta:root","control_pool":"b2.rgw.control","dedup_pool":"b2.rgw.dedup","gc_pool":"b2.rgw.log:gc","lc_pool":"b2.rgw.log:lc","log_pool":"b2.rgw.log","intent_log_pool":"b2.rgw.log:intent","usage_log_pool":"b2.rgw.log:usage","roles_pool":"b2.rgw.meta:roles","reshard_pool":"b2.rgw.log:reshard","user_keys_pool":"b2.rgw.meta:users.keys","user_email_pool":"b2.rgw.meta:users.email","user_swift_pool":"b2.rgw.meta:users.swift","user_uid_pool":"b2.rgw.meta:users.uid","otp_pool":"b2.rgw.otp","notif_pool":"b2.rgw.log:notif","topics_pool":"b2.rgw.meta:topics","account_pool":"b2.rgw.meta:accounts","group_pool":"b2.rgw.meta:groups","system_key":{"access_key":"GRL33KL1ULN5795C","secret_key":"FPGvuVSJOG1341zbJPfw7ch710f3izIf"},"placement_pools":[{"key":"default-placement","val":{"index_pool":"b2.rgw.buckets.index","storage_classes":{"STANDARD":{"data_pool":"b2.rgw.buckets.data"}},"data_extra_pool":"b2.rgw.buckets.non-ec","index_type":0,"inline_data":true}}],"realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","restore_pool":"b2.rgw.log:restore"} 2026-03-20T11:24:38.664 DEBUG:teuthology.orchestra.run.vm08:> sudo ceph osd pool create b2.rgw.buckets.data 64 64 --cluster c2 2026-03-20T11:24:39.575 INFO:teuthology.orchestra.run.vm08.stderr:pool 'b2.rgw.buckets.data' created 2026-03-20T11:24:39.601 DEBUG:teuthology.orchestra.run.vm08:> sudo ceph osd pool application enable b2.rgw.buckets.data rgw --cluster c2 2026-03-20T11:24:40.577 INFO:teuthology.orchestra.run.vm08.stderr:enabled application 'rgw' on pool 'b2.rgw.buckets.data' 2026-03-20T11:24:40.608 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['zone', 'placement', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', 'modify', '--placement-id', 'default-placement', '--compression', 'random', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:24:40.608 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'zone', 'placement', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', 'modify', '--placement-id', 'default-placement', '--compression', 'random', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:24:40.608 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 zone placement --rgw-zone b2 --rgw-zonegroup b --rgw-realm test-realm modify --placement-id default-placement --compression random --cluster c2 --debug-rgw 0 --debug-ms 0 2026-03-20T11:24:40.652 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:24:40.653 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:24:40.764 INFO:teuthology.orchestra.run.vm07.stdout:{"id":"cd52b229-0d4a-43bf-b4ba-156aa19e7aef","name":"b2","domain_root":"b2.rgw.meta:root","control_pool":"b2.rgw.control","dedup_pool":"b2.rgw.dedup","gc_pool":"b2.rgw.log:gc","lc_pool":"b2.rgw.log:lc","log_pool":"b2.rgw.log","intent_log_pool":"b2.rgw.log:intent","usage_log_pool":"b2.rgw.log:usage","roles_pool":"b2.rgw.meta:roles","reshard_pool":"b2.rgw.log:reshard","user_keys_pool":"b2.rgw.meta:users.keys","user_email_pool":"b2.rgw.meta:users.email","user_swift_pool":"b2.rgw.meta:users.swift","user_uid_pool":"b2.rgw.meta:users.uid","otp_pool":"b2.rgw.otp","notif_pool":"b2.rgw.log:notif","topics_pool":"b2.rgw.meta:topics","account_pool":"b2.rgw.meta:accounts","group_pool":"b2.rgw.meta:groups","system_key":{"access_key":"GRL33KL1ULN5795C","secret_key":"FPGvuVSJOG1341zbJPfw7ch710f3izIf"},"placement_pools":[{"key":"default-placement","val":{"index_pool":"b2.rgw.buckets.index","storage_classes":{"STANDARD":{"data_pool":"b2.rgw.buckets.data","compression_type":"random"}},"data_extra_pool":"b2.rgw.buckets.non-ec","index_type":0,"inline_data":true}}],"realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","restore_pool":"b2.rgw.log:restore"} 2026-03-20T11:24:40.764 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['period', 'update', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:24:40.764 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'period', 'update', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:24:40.765 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 period update --rgw-zone b2 --rgw-zonegroup b --rgw-realm test-realm --commit --cluster c2 --debug-rgw 0 --debug-ms 0 2026-03-20T11:24:40.805 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:24:40.805 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:24:40.827 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:24:40.828+0000 7fa332939900 0 period (cb2fcdce-2c1b-429a-8601-a284ee3a2381 does not have zone cd52b229-0d4a-43bf-b4ba-156aa19e7aef configured 2026-03-20T11:24:47.788 INFO:teuthology.orchestra.run.vm07.stderr:Sending period to new master zone 4b634ebf-58f5-45bc-839a-cb4b425e773a 2026-03-20T11:24:47.834 INFO:teuthology.orchestra.run.vm07.stdout:{"id":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","epoch":4,"predecessor_uuid":"5d4b2b15-cd70-4a63-87c8-e59369f9ab35","sync_status":[],"period_map":{"id":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","zonegroups":[{"id":"a","name":"a","api_name":"a","is_master":true,"endpoints":["http://vm02.local:8000"],"hostnames":[],"hostnames_s3website":[],"master_zone":"4b634ebf-58f5-45bc-839a-cb4b425e773a","zones":[{"id":"4b634ebf-58f5-45bc-839a-cb4b425e773a","name":"a1","endpoints":["http://vm02.local:8000"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]},{"id":"51055576-6662-409d-8abb-ecaeefd3dd80","name":"a2","endpoints":["http://vm05.local:8001"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]}],"placement_targets":[{"name":"default-placement","tags":[],"storage_classes":["STANDARD"]}],"default_placement":"default-placement","realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","sync_policy":{"groups":[]},"enabled_features":["notification_v2","resharding"]},{"id":"b","name":"b","api_name":"b","is_master":false,"endpoints":["http://vm07.local:8000"],"hostnames":[],"hostnames_s3website":[],"master_zone":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","zones":[{"id":"cd52b229-0d4a-43bf-b4ba-156aa19e7aef","name":"b2","endpoints":["http://vm08.local:8001"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]},{"id":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","name":"b1","endpoints":["http://vm07.local:8000"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]}],"placement_targets":[{"name":"default-placement","tags":[],"storage_classes":["STANDARD"]}],"default_placement":"default-placement","realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","sync_policy":{"groups":[]},"enabled_features":["notification_v2","resharding"]}],"short_zone_ids":[{"key":"4b634ebf-58f5-45bc-839a-cb4b425e773a","val":162505008},{"key":"51055576-6662-409d-8abb-ecaeefd3dd80","val":2776933003},{"key":"cd52b229-0d4a-43bf-b4ba-156aa19e7aef","val":2686426109},{"key":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","val":114662526}]},"master_zonegroup":"a","master_zone":"4b634ebf-58f5-45bc-839a-cb4b425e773a","period_config":{"bucket_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false},"bucket_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false},"anonymous_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false}},"realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","realm_epoch":2} 2026-03-20T11:24:47.835 DEBUG:tasks.rgw.c2.client.1:waiting for process to exit 2026-03-20T11:24:47.835 INFO:teuthology.orchestra.run:waiting for 300 2026-03-20T11:24:47.835 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:24:47.837+0000 7fcc35488640 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper term radosgw --rgw-frontends beast port=8001 -n client.1 --cluster c2 -k /etc/ceph/c2.client.1.keyring --log-file /var/log/ceph/rgw.c2.client.1.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c2.client.1.sock --foreground (PID: 56674) UID: 0 2026-03-20T11:24:47.835 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:24:47.837+0000 7fcc38f2a980 -1 shutting down 2026-03-20T11:25:05.100 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.1 is failed for ~0s 2026-03-20T11:25:05.280 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:25:05.282+0000 7f765b3a8640 -1 req 12327821371654137473 0.003000008s :post_period failed to decode period 2026-03-20T11:25:06.139 INFO:tasks.rgw.c2.client.1:Stopped 2026-03-20T11:25:06.139 INFO:tasks.rgw.c2.client.1:Restarting daemon 2026-03-20T11:25:06.139 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8001' -n client.1 --cluster c2 -k /etc/ceph/c2.client.1.keyring --log-file /var/log/ceph/rgw.c2.client.1.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c2.client.1.sock --foreground --rgw-zone b2 --rgw-zonegroup b --rgw-realm test-realm | sudo tee /var/log/ceph/rgw.c2.client.1.stdout 2>&1 2026-03-20T11:25:06.141 INFO:tasks.rgw.c2.client.1:Started 2026-03-20T11:25:06.141 DEBUG:teuthology.orchestra.run.vm08:> curl http://vm08.local:8001 2026-03-20T11:25:06.161 DEBUG:teuthology.orchestra.run:got remote process result: 7 2026-03-20T11:25:06.161 INFO:teuthology.orchestra.run.vm08.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2026-03-20T11:25:06.161 INFO:teuthology.orchestra.run.vm08.stderr: Dload Upload Total Spent Left Speed 2026-03-20T11:25:06.161 INFO:teuthology.orchestra.run.vm08.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 2026-03-20T11:25:06.161 INFO:teuthology.orchestra.run.vm08.stderr:curl: (7) Failed to connect to vm08.local port 8001: Connection refused 2026-03-20T11:25:07.162 DEBUG:teuthology.orchestra.run.vm08:> curl http://vm08.local:8001 2026-03-20T11:25:07.185 INFO:teuthology.orchestra.run.vm08.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2026-03-20T11:25:07.185 INFO:teuthology.orchestra.run.vm08.stderr: Dload Upload Total Spent Left Speed 2026-03-20T11:25:07.186 INFO:teuthology.orchestra.run.vm08.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 187 0 187 0 0 62333 0 --:--:-- --:--:-- --:--:-- 62333 2026-03-20T11:25:07.187 INFO:teuthology.orchestra.run.vm08.stdout:anonymous 2026-03-20T11:25:07.187 INFO:tasks.rgw_multisite:rgw multisite configuration completed 2026-03-20T11:25:07.187 INFO:teuthology.run_tasks:Running task rgw-multisite-tests... 2026-03-20T11:25:07.190 INFO:tasks.rgw_multisite_tests:importing tests from /home/teuthos/src/github.com_kshtsk_ceph_7b4fb1902b22ff4ea3f4ff6a953bc42198ebeffe/qa/../src/test/rgw/rgw_multi 2026-03-20T11:25:07.210 INFO:tasks.rgw_multisite_tests:creating test user.. 2026-03-20T11:25:07.211 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['account', 'create', '--account-id', 'RGW11111111111111111', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:25:07.211 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'account', 'create', '--account-id', 'RGW11111111111111111', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:25:07.211 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 account create --account-id RGW11111111111111111 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:25:07.247 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:25:07.247 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:25:07.325 INFO:teuthology.orchestra.run.vm02.stdout:{"id":"RGW11111111111111111","tenant":"","name":"","email":"","quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"bucket_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"max_users":1000,"max_roles":1000,"max_groups":1000,"max_buckets":1000,"max_access_keys":4} 2026-03-20T11:25:07.325 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['user', 'create', '--uid', 'rgw-multisite-test-user', '--account-id', 'RGW11111111111111111', '--account-root', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--display-name', 'TestUser', '--gen-access-key', '--gen-secret', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:25:07.325 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'user', 'create', '--uid', 'rgw-multisite-test-user', '--account-id', 'RGW11111111111111111', '--account-root', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--display-name', 'TestUser', '--gen-access-key', '--gen-secret', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:25:07.325 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 user create --uid rgw-multisite-test-user --account-id RGW11111111111111111 --account-root --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --display-name TestUser --gen-access-key --gen-secret --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:25:07.405 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:25:07.405 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:25:07.478 INFO:teuthology.orchestra.run.vm02.stdout:{"user_id":"rgw-multisite-test-user","display_name":"TestUser","email":"","suspended":0,"max_buckets":1000,"subusers":[],"keys":[{"user":"rgw-multisite-test-user","access_key":"250ZKEJ67E9N42C4RBZK","secret_key":"JuXQXbM8rZ4J6fBYIR3Px2CAJEdkRft9GJeITxWI","active":true,"create_date":"2026-03-20T11:25:07.475252Z"}],"swift_keys":[],"caps":[],"op_mask":"read, write, delete","default_placement":"","default_storage_class":"","placement_tags":[],"bucket_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"temp_url_keys":[],"type":"root","mfa_ids":[],"account_id":"RGW11111111111111111","path":"/","create_date":"2026-03-20T11:25:07.475042Z","tags":[],"group_ids":[]} 2026-03-20T11:25:07.488 INFO:tasks.rgw_multisite_tests:creating non-account user.. 2026-03-20T11:25:07.488 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['user', 'create', '--uid', 'rgw-multisite-test-non-account-user', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--display-name', 'NonAccountUser', '--gen-access-key', '--gen-secret', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:25:07.488 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'user', 'create', '--uid', 'rgw-multisite-test-non-account-user', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--display-name', 'NonAccountUser', '--gen-access-key', '--gen-secret', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:25:07.488 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 user create --uid rgw-multisite-test-non-account-user --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --display-name NonAccountUser --gen-access-key --gen-secret --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:25:07.568 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:25:07.568 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:25:07.643 INFO:teuthology.orchestra.run.vm02.stdout:{"user_id":"rgw-multisite-test-non-account-user","display_name":"NonAccountUser","email":"","suspended":0,"max_buckets":1000,"subusers":[],"keys":[{"user":"rgw-multisite-test-non-account-user","access_key":"H2Z3CE0C7NE0W6WMX2NG","secret_key":"JshxFZtiibfUDWBdCP4omVEJA3gRMSBN4urIvDu7","active":true,"create_date":"2026-03-20T11:25:07.641188Z"}],"swift_keys":[],"caps":[],"op_mask":"read, write, delete","default_placement":"","default_storage_class":"","placement_tags":[],"bucket_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"temp_url_keys":[],"type":"rgw","mfa_ids":[],"account_id":"","path":"/","create_date":"2026-03-20T11:25:07.640907Z","tags":[],"group_ids":[]} 2026-03-20T11:25:07.652 INFO:tasks.rgw_multisite_tests:creating non-account alt user.. 2026-03-20T11:25:07.652 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['user', 'create', '--uid', 'rgw-multisite-test-non-account-alt-user', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--display-name', 'NonAccountAltUser', '--gen-access-key', '--gen-secret', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:25:07.652 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'user', 'create', '--uid', 'rgw-multisite-test-non-account-alt-user', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--display-name', 'NonAccountAltUser', '--gen-access-key', '--gen-secret', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:25:07.652 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 user create --uid rgw-multisite-test-non-account-alt-user --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --display-name NonAccountAltUser --gen-access-key --gen-secret --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:25:07.733 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:25:07.733 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:25:07.808 INFO:teuthology.orchestra.run.vm02.stdout:{"user_id":"rgw-multisite-test-non-account-alt-user","display_name":"NonAccountAltUser","email":"","suspended":0,"max_buckets":1000,"subusers":[],"keys":[{"user":"rgw-multisite-test-non-account-alt-user","access_key":"E6UBSZ7XYA30035YYKC9","secret_key":"nqnOHzLmXNYFX2BHOHd161cMrGEYkuSBLCNuBsLd","active":true,"create_date":"2026-03-20T11:25:07.806021Z"}],"swift_keys":[],"caps":[],"op_mask":"read, write, delete","default_placement":"","default_storage_class":"","placement_tags":[],"bucket_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"temp_url_keys":[],"type":"rgw","mfa_ids":[],"account_id":"","path":"/","create_date":"2026-03-20T11:25:07.805788Z","tags":[],"group_ids":[]} 2026-03-20T11:25:07.817 INFO:rgw_multi.tests:meta checkpoint 2026-03-20T11:25:07.817 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:25:07.817 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:25:07.818 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:25:07.894 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:25:07.894 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:25:07.960 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:25:07.970 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774005907.477197_41.1","last_update":"2026-03-20T11:25:07.477197Z"},{"marker":"","last_update":"0.000000"},{"marker":"1_1774005907.643115_44.1","last_update":"2026-03-20T11:25:07.643115Z"},{"marker":"1_1774005907.807893_68.1","last_update":"2026-03-20T11:25:07.807893Z"}] 2026-03-20T11:25:07.971 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:25:07.971 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:25:07.971 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:25:07.971 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:25:08.047 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:25:08.048 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:25:08.119 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:25:08.120+0000 7f63ff210900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:25:08.119 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:25:08.121+0000 7f63ff210900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:25:08.130 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:25:08Z"} 2026-03-20T11:25:08.130 WARNING:rgw_multi.tests:zone a2 behind master: shard=0 master=1_1774005907.477197_41.1 target=, shard=2 master=1_1774005907.643115_44.1 target=, shard=3 master=1_1774005907.807893_68.1 target= 2026-03-20T11:25:11.106 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.1 has been restored 2026-03-20T11:25:13.131 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:25:13.131 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:25:13.131 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:25:13.169 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:25:13.169 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:25:13.238 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:25:13.240+0000 7f8085339900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:25:13.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:25:13.241+0000 7f8085339900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:25:13.249 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:25:13Z"} 2026-03-20T11:25:13.249 WARNING:rgw_multi.tests:zone a2 behind master: shard=0 master=1_1774005907.477197_41.1 target=, shard=2 master=1_1774005907.643115_44.1 target=, shard=3 master=1_1774005907.807893_68.1 target= 2026-03-20T11:25:18.250 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:25:18.250 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:25:18.250 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:25:18.285 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:25:18.285 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:25:18.357 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:25:18.358+0000 7ff00cd21900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:25:18.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:25:18.359+0000 7ff00cd21900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:25:18.368 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:25:18Z"} 2026-03-20T11:25:18.368 WARNING:rgw_multi.tests:zone a2 behind master: shard=0 master=1_1774005907.477197_41.1 target=, shard=2 master=1_1774005907.643115_44.1 target=, shard=3 master=1_1774005907.807893_68.1 target= 2026-03-20T11:25:23.369 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:25:23.369 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:25:23.369 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:25:23.410 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:25:23.410 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:25:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:25:23.489+0000 7f89b4f26900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:25:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:25:23.490+0000 7f89b4f26900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:25:23.499 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:25:23Z"} 2026-03-20T11:25:23.499 WARNING:rgw_multi.tests:zone a2 behind master: shard=0 master=1_1774005907.477197_41.1 target=, shard=2 master=1_1774005907.643115_44.1 target=, shard=3 master=1_1774005907.807893_68.1 target= 2026-03-20T11:25:28.500 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:25:28.500 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:25:28.500 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:25:28.542 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:25:28.542 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:25:28.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:25:28.617+0000 7f9ab3612900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:25:28.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:25:28.618+0000 7f9ab3612900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:25:28.627 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:25:28Z"} 2026-03-20T11:25:28.627 WARNING:rgw_multi.tests:zone a2 behind master: shard=0 master=1_1774005907.477197_41.1 target=, shard=2 master=1_1774005907.643115_44.1 target=, shard=3 master=1_1774005907.807893_68.1 target= 2026-03-20T11:25:33.628 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:25:33.628 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:25:33.628 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:25:33.667 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:25:33.667 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:25:33.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:25:33.740+0000 7f9901326900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:25:33.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:25:33.741+0000 7f9901326900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:25:33.750 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:25:33Z"} 2026-03-20T11:25:33.750 WARNING:rgw_multi.tests:zone a2 behind master: shard=0 master=1_1774005907.477197_41.1 target=, shard=2 master=1_1774005907.643115_44.1 target=, shard=3 master=1_1774005907.807893_68.1 target= 2026-03-20T11:25:38.750 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:25:38.751 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:25:38.751 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:25:38.787 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:25:38.787 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:25:38.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:25:38.855+0000 7f29dce20900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:25:38.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:25:38.856+0000 7f29dce20900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:25:38.864 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:25:38Z"} 2026-03-20T11:25:38.864 WARNING:rgw_multi.tests:zone a2 behind master: shard=0 master=1_1774005907.477197_41.1 target=, shard=2 master=1_1774005907.643115_44.1 target=, shard=3 master=1_1774005907.807893_68.1 target= 2026-03-20T11:25:43.864 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:25:43.865 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:25:43.865 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:25:43.901 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:25:43.901 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:25:43.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:25:43.971+0000 7fea63341900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:25:43.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:25:43.972+0000 7fea63341900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:25:43.980 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:25:43Z"} 2026-03-20T11:25:43.980 WARNING:rgw_multi.tests:zone a2 behind master: shard=0 master=1_1774005907.477197_41.1 target=, shard=2 master=1_1774005907.643115_44.1 target=, shard=3 master=1_1774005907.807893_68.1 target= 2026-03-20T11:25:48.981 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:25:48.981 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:25:48.981 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:25:49.020 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:25:49.020 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:25:49.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:25:49.095+0000 7f840481c900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:25:49.095 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:25:49.096+0000 7f840481c900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:25:49.105 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:25:49Z"} 2026-03-20T11:25:49.105 WARNING:rgw_multi.tests:zone a2 behind master: shard=0 master=1_1774005907.477197_41.1 target=, shard=2 master=1_1774005907.643115_44.1 target=, shard=3 master=1_1774005907.807893_68.1 target= 2026-03-20T11:25:54.105 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:25:54.105 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:25:54.105 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:25:54.143 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:25:54.144 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:25:54.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:25:54.208+0000 7fde2101d900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:25:54.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:25:54.209+0000 7fde2101d900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:25:54.218 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:25:54Z"} 2026-03-20T11:25:54.218 WARNING:rgw_multi.tests:zone a2 behind master: shard=0 master=1_1774005907.477197_41.1 target=, shard=2 master=1_1774005907.643115_44.1 target=, shard=3 master=1_1774005907.807893_68.1 target= 2026-03-20T11:25:59.219 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:25:59.219 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:25:59.219 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:25:59.255 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:25:59.255 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:25:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:25:59.319+0000 7ffb9fc10900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:25:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:25:59.319+0000 7ffb9fc10900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:25:59.327 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:25:59Z"} 2026-03-20T11:25:59.327 WARNING:rgw_multi.tests:zone a2 behind master: shard=0 master=1_1774005907.477197_41.1 target=, shard=2 master=1_1774005907.643115_44.1 target=, shard=3 master=1_1774005907.807893_68.1 target= 2026-03-20T11:26:04.327 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:04.328 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:04.328 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:04.362 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:04.362 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:04.432 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:04.434+0000 7fca89c12900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:04.433 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:04.434+0000 7fca89c12900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:04.442 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:26:04Z"} 2026-03-20T11:26:04.442 WARNING:rgw_multi.tests:zone a2 behind master: shard=0 master=1_1774005907.477197_41.1 target=, shard=2 master=1_1774005907.643115_44.1 target=, shard=3 master=1_1774005907.807893_68.1 target= 2026-03-20T11:26:09.443 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:09.444 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:09.444 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:09.477 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:09.477 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:09.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:09.540+0000 7fd12292f900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:09.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:09.540+0000 7fd12292f900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:09.548 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:26:09Z"} 2026-03-20T11:26:09.548 WARNING:rgw_multi.tests:zone a2 behind master: shard=0 master=1_1774005907.477197_41.1 target=, shard=2 master=1_1774005907.643115_44.1 target=, shard=3 master=1_1774005907.807893_68.1 target= 2026-03-20T11:26:14.549 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:14.549 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:14.549 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:14.584 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:14.585 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:14.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:14.645+0000 7f6ada14d900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:14.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:14.646+0000 7f6ada14d900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:14.654 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:26:14Z"} 2026-03-20T11:26:14.655 WARNING:rgw_multi.tests:zone a2 behind master: shard=0 master=1_1774005907.477197_41.1 target=, shard=2 master=1_1774005907.643115_44.1 target=, shard=3 master=1_1774005907.807893_68.1 target= 2026-03-20T11:26:19.655 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:19.655 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:19.655 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:19.695 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:19.695 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:19.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:19.759+0000 7f099a922900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:19.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:19.759+0000 7f099a922900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:19.767 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:26:19Z"} 2026-03-20T11:26:19.767 WARNING:rgw_multi.tests:zone a2 behind master: shard=0 master=1_1774005907.477197_41.1 target=, shard=2 master=1_1774005907.643115_44.1 target=, shard=3 master=1_1774005907.807893_68.1 target= 2026-03-20T11:26:24.767 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:24.768 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:24.768 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:24.805 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:24.805 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:24.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:24.866+0000 7f6f53fcc900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:24.866 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:24.867+0000 7f6f53fcc900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:24.875 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:26:24Z"} 2026-03-20T11:26:24.875 WARNING:rgw_multi.tests:zone a2 behind master: shard=0 master=1_1774005907.477197_41.1 target=, shard=2 master=1_1774005907.643115_44.1 target=, shard=3 master=1_1774005907.807893_68.1 target= 2026-03-20T11:26:29.876 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:29.876 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:29.876 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:29.913 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:29.913 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:29.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:29.972+0000 7f9b3f341900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:29.972 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:29.973+0000 7f9b3f341900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:29.981 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:26:29Z"} 2026-03-20T11:26:29.981 WARNING:rgw_multi.tests:zone a2 behind master: shard=0 master=1_1774005907.477197_41.1 target=, shard=2 master=1_1774005907.643115_44.1 target=, shard=3 master=1_1774005907.807893_68.1 target= 2026-03-20T11:26:34.982 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:34.982 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:34.982 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:35.020 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:35.020 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:35.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:35.087+0000 7fd8c6528900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:35.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:35.087+0000 7fd8c6528900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:35.094 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:26:35Z"} 2026-03-20T11:26:35.094 WARNING:rgw_multi.tests:zone a2 behind master: shard=0 master=1_1774005907.477197_41.1 target=, shard=2 master=1_1774005907.643115_44.1 target=, shard=3 master=1_1774005907.807893_68.1 target= 2026-03-20T11:26:40.095 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:40.096 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:40.096 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:40.133 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:40.133 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:40.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:40.197+0000 7f7dd7f2a900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:40.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:40.198+0000 7f7dd7f2a900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:40.205 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774005907.477197_41.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:25:07.477197Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774005907.643115_44.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:25:07.643115Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774005907.807893_68.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:25:07.807893Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:26:40Z"} 2026-03-20T11:26:40.206 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:26:40.206 INFO:rgw_multi.tests:starting meta checkpoint for zone=b1 2026-03-20T11:26:40.206 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:40.206 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:40.206 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:40.245 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:40.245 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:40.306 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:26:40.307+0000 7f8be172d900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:40.306 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:26:40.308+0000 7f8be172d900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:40.315 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774005907.477197_41.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:25:07.477197Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774005907.643115_44.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:25:07.643115Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774005907.807893_68.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:25:07.807893Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:26:40Z"} 2026-03-20T11:26:40.315 INFO:rgw_multi.tests:finish meta checkpoint for zone=b1 2026-03-20T11:26:40.315 INFO:rgw_multi.tests:starting meta checkpoint for zone=b2 2026-03-20T11:26:40.315 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:40.315 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:40.315 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b2 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:40.392 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:40.392 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:40.453 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:26:40.455+0000 7f7088410900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:40.454 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:26:40.456+0000 7f7088410900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:40.463 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774005907.477197_41.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:25:07.477197Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774005907.643115_44.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:25:07.643115Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774005907.807893_68.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:25:07.807893Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:26:40Z"} 2026-03-20T11:26:40.463 INFO:rgw_multi.tests:finish meta checkpoint for zone=b2 2026-03-20T11:26:40.463 INFO:rgw_multi.tests:meta checkpoint 2026-03-20T11:26:40.463 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:40.463 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:40.463 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:40.497 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:40.497 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:40.560 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:26:40.569 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774005907.477197_41.1","last_update":"2026-03-20T11:25:07.477197Z"},{"marker":"","last_update":"0.000000"},{"marker":"1_1774005907.643115_44.1","last_update":"2026-03-20T11:25:07.643115Z"},{"marker":"1_1774005907.807893_68.1","last_update":"2026-03-20T11:25:07.807893Z"}] 2026-03-20T11:26:40.569 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:26:40.569 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:40.569 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:40.569 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:40.646 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:40.646 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:40.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:40.711+0000 7fb4f3620900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:40.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:40.712+0000 7fb4f3620900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:40.720 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774005907.477197_41.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:25:07.477197Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774005907.643115_44.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:25:07.643115Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774005907.807893_68.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:25:07.807893Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:26:40Z"} 2026-03-20T11:26:40.720 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:26:40.720 INFO:rgw_multi.tests:starting meta checkpoint for zone=b1 2026-03-20T11:26:40.721 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:40.721 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:40.721 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:40.762 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:40.762 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:40.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:26:40.824+0000 7fc889b41900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:40.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:26:40.825+0000 7fc889b41900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:40.832 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774005907.477197_41.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:25:07.477197Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774005907.643115_44.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:25:07.643115Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774005907.807893_68.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:25:07.807893Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:26:40Z"} 2026-03-20T11:26:40.833 INFO:rgw_multi.tests:finish meta checkpoint for zone=b1 2026-03-20T11:26:40.833 INFO:rgw_multi.tests:starting meta checkpoint for zone=b2 2026-03-20T11:26:40.833 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:40.833 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:40.833 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b2 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:40.909 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:40.909 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:40.970 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:26:40.972+0000 7f87c6a1a900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:40.971 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:26:40.973+0000 7f87c6a1a900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:40.981 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774005907.477197_41.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:25:07.477197Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774005907.643115_44.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:25:07.643115Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774005907.807893_68.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:25:07.807893Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:26:40Z"} 2026-03-20T11:26:40.981 INFO:rgw_multi.tests:finish meta checkpoint for zone=b2 2026-03-20T11:26:40.981 INFO:tasks.rgw_multisite_tests:running rgw multisite tests on '/home/teuthos/src/github.com_kshtsk_ceph_7b4fb1902b22ff4ea3f4ff6a953bc42198ebeffe/qa/../src/test/rgw/rgw_multi' with args=['tests.py', '-a', '!fails_with_rgw'] 2026-03-20T11:26:41.242 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:41.242 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:41.242 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:41.276 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:41.277 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:41.336 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:26:41.345 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774005907.477197_41.1","last_update":"2026-03-20T11:25:07.477197Z"},{"marker":"","last_update":"0.000000"},{"marker":"1_1774005907.643115_44.1","last_update":"2026-03-20T11:25:07.643115Z"},{"marker":"1_1774005907.807893_68.1","last_update":"2026-03-20T11:25:07.807893Z"}] 2026-03-20T11:26:41.345 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:26:41.345 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:41.346 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:41.346 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:41.422 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:41.422 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:41.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:41.486+0000 7fa14914b900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:41.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:41.487+0000 7fa14914b900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:41.495 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774005907.477197_41.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:25:07.477197Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774005907.643115_44.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:25:07.643115Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774005907.807893_68.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:25:07.807893Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:26:41Z"} 2026-03-20T11:26:41.495 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:26:41.495 INFO:rgw_multi.tests:create bucket zone=a1 name=dipepu-1 2026-03-20T11:26:43.827 INFO:rgw_multi.tests:create bucket zone=a2 name=dipepu-2 2026-03-20T11:26:46.845 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:46.845 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:46.845 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:46.882 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:46.882 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:46.948 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:26:46.957 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006003.844578_43.1","last_update":"2026-03-20T11:26:43.844578Z"},{"marker":"","last_update":"0.000000"},{"marker":"1_1774005907.643115_44.1","last_update":"2026-03-20T11:25:07.643115Z"},{"marker":"1_1774006003.847015_70.1","last_update":"2026-03-20T11:26:43.847015Z"}] 2026-03-20T11:26:46.957 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:26:46.957 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:46.957 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:46.957 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:47.032 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:47.032 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:47.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:47.098+0000 7f5790b4b900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:47.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:47.099+0000 7f5790b4b900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:47.107 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006003.844578_43.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:43.844578Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774005907.643115_44.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:25:07.643115Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006003.847015_70.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:26:43.847015Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:26:47Z"} 2026-03-20T11:26:47.107 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:26:47.116 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_create ... ok 2026-03-20T11:26:47.194 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['user', 'create', '--tenant', 'testx', '--uid', 'test', '--access-key', 'abcd', '--secret-key', 'efgh', '--display-name', 'tenanted-user', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:26:47.194 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'user', 'create', '--tenant', 'testx', '--uid', 'test', '--access-key', 'abcd', '--secret-key', 'efgh', '--display-name', 'tenanted-user', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:26:47.194 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 user create --tenant testx --uid test --access-key abcd --secret-key efgh --display-name tenanted-user --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:26:47.234 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:47.234 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:47.300 INFO:teuthology.orchestra.run.vm02.stdout:{"user_id":"testx$test","display_name":"tenanted-user","email":"","suspended":0,"max_buckets":1000,"subusers":[],"keys":[{"user":"testx$test","access_key":"abcd","secret_key":"efgh","active":true,"create_date":"2026-03-20T11:26:47.299057Z"}],"swift_keys":[],"caps":[],"op_mask":"read, write, delete","default_placement":"","default_storage_class":"","placement_tags":[],"bucket_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"temp_url_keys":[],"type":"rgw","mfa_ids":[],"account_id":"","path":"/","create_date":"2026-03-20T11:26:47.299052Z","tags":[],"group_ids":[]} 2026-03-20T11:26:47.309 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:47.309 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:47.309 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:47.385 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:47.385 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:47.448 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:26:47.457 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006003.844578_43.1","last_update":"2026-03-20T11:26:43.844578Z"},{"marker":"1_1774006007.300792_36.1","last_update":"2026-03-20T11:26:47.300792Z"},{"marker":"1_1774005907.643115_44.1","last_update":"2026-03-20T11:25:07.643115Z"},{"marker":"1_1774006003.847015_70.1","last_update":"2026-03-20T11:26:43.847015Z"}] 2026-03-20T11:26:47.457 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:26:47.457 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:47.457 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:47.458 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:47.533 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:47.534 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:47.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:47.597+0000 7f8d8ba13900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:47.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:47.598+0000 7f8d8ba13900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:47.606 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006003.844578_43.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:43.844578Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"0.000000","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774005907.643115_44.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:25:07.643115Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006003.847015_70.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:26:43.847015Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:26:47Z"} 2026-03-20T11:26:47.606 WARNING:rgw_multi.tests:zone a2 behind master: shard=1 master=1_1774006007.300792_36.1 target= 2026-03-20T11:26:52.607 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:52.607 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:52.607 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:52.642 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:52.642 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:52.704 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:52.705+0000 7fdd95e20900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:52.705 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:52.706+0000 7fdd95e20900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:52.713 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006003.844578_43.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:43.844578Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006007.300792_36.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:26:47.300792Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774005907.643115_44.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:25:07.643115Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006003.847015_70.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:26:43.847015Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:26:52Z"} 2026-03-20T11:26:52.713 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:26:52.735 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:52.735 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:52.735 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:52.786 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:52.786 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:52.853 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:26:52.865 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006012.724467_44.1","last_update":"2026-03-20T11:26:52.724467Z"},{"marker":"1_1774006007.300792_36.1","last_update":"2026-03-20T11:26:47.300792Z"},{"marker":"1_1774005907.643115_44.1","last_update":"2026-03-20T11:25:07.643115Z"},{"marker":"1_1774006012.727259_71.1","last_update":"2026-03-20T11:26:52.727259Z"}] 2026-03-20T11:26:52.865 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:26:52.865 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:52.865 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:52.866 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:52.940 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:52.940 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:53.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:53.009+0000 7f035a952900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:53.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:53.009+0000 7f035a952900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:53.018 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006012.724467_44.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:52.724467Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006007.300792_36.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:26:47.300792Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774005907.643115_44.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:25:07.643115Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006012.727259_71.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:26:52.727259Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:26:53Z"} 2026-03-20T11:26:53.018 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:26:53.020 INFO:rgw_multi.tests:bucket exists in tenant namespace 2026-03-20T11:26:53.023 INFO:rgw_multi.tests:bucket does not exist in default user namespace 2026-03-20T11:26:53.023 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['user', 'rm', '--tenant', 'testx', '--uid', 'test', '--purge-data', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:26:53.023 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'user', 'rm', '--tenant', 'testx', '--uid', 'test', '--purge-data', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:26:53.023 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 user rm --tenant testx --uid test --purge-data --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:26:53.094 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:53.094 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:53.259 INFO:tasks.rgw_multisite_tests:create a bucket from secondary zone under tenant namespace. check if it successfully syncs ... ok 2026-03-20T11:26:53.342 INFO:rgw_multi.tests:create bucket zone=a1 name=dipepu-3 2026-03-20T11:26:53.355 INFO:rgw_multi.tests:create bucket zone=a2 name=dipepu-4 2026-03-20T11:26:53.377 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:53.377 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:53.377 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:53.412 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:53.412 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:53.493 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:26:53.508 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006013.366294_47.1","last_update":"2026-03-20T11:26:53.366294Z"},{"marker":"1_1774006013.250425_37.1","last_update":"2026-03-20T11:26:53.250425Z"},{"marker":"1_1774005907.643115_44.1","last_update":"2026-03-20T11:25:07.643115Z"},{"marker":"1_1774006013.368869_75.1","last_update":"2026-03-20T11:26:53.368869Z"}] 2026-03-20T11:26:53.508 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:26:53.508 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:53.508 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:53.509 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:53.586 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:53.586 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:53.651 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:53.652+0000 7fb7a192f900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:53.651 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:53.653+0000 7fb7a192f900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:53.660 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006013.366294_47.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:53.366294Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006007.300792_36.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:26:47.300792Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774005907.643115_44.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:25:07.643115Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006013.368869_75.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:26:53.368869Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:26:53Z"} 2026-03-20T11:26:53.660 WARNING:rgw_multi.tests:zone a2 behind master: shard=1 master=1_1774006013.250425_37.1 target=1_1774006007.300792_36.1 2026-03-20T11:26:55.852 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:26:55.854+0000 7f11de8ad640 -1 WARNING: The OSD cluster is overloaded and struggling to complete ops. You need more capacity to serve this level of demand. 2026-03-20T11:26:58.661 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:58.661 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:58.661 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:58.695 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:58.695 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:58.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:58.758+0000 7f4005f3d900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:58.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:58.759+0000 7f4005f3d900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:58.767 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006013.366294_47.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:53.366294Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006013.250425_37.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:26:53.250425Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774005907.643115_44.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:25:07.643115Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006013.368869_75.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:26:53.368869Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:26:58Z"} 2026-03-20T11:26:58.767 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:26:58.891 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:58.891 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:58.891 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:58.928 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:58.928 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:58.993 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:26:59.004 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006013.250425_37.1","last_update":"2026-03-20T11:26:53.250425Z"},{"marker":"1_1774005907.643115_44.1","last_update":"2026-03-20T11:25:07.643115Z"},{"marker":"1_1774006013.368869_75.1","last_update":"2026-03-20T11:26:53.368869Z"}] 2026-03-20T11:26:59.004 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:26:59.004 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:59.004 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:59.004 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:59.079 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:59.079 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:59.142+0000 7efd5c723900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:59.143+0000 7efd5c723900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:59.150 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006013.250425_37.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:26:53.250425Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774005907.643115_44.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:25:07.643115Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006013.368869_75.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:26:53.368869Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:26:59Z"} 2026-03-20T11:26:59.151 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:26:59.158 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_recreate ... ok 2026-03-20T11:26:59.203 INFO:rgw_multi.tests:create bucket zone=a1 name=dipepu-5 2026-03-20T11:26:59.216 INFO:rgw_multi.tests:create bucket zone=a2 name=dipepu-6 2026-03-20T11:26:59.235 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:59.235 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:59.235 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:59.272 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:59.272 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:59.336 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:26:59.345 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006019.226920_42.1","last_update":"2026-03-20T11:26:59.226920Z"},{"marker":"1_1774005907.643115_44.1","last_update":"2026-03-20T11:25:07.643115Z"},{"marker":"1_1774006019.229407_77.1","last_update":"2026-03-20T11:26:59.229407Z"}] 2026-03-20T11:26:59.345 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:26:59.345 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:59.345 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:59.345 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:59.421 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:59.421 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:59.480 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:59.482+0000 7f39c7739900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:59.481 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:59.482+0000 7f39c7739900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:59.490 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006019.226920_42.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:26:59.226920Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774005907.643115_44.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:25:07.643115Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006019.229407_77.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:26:59.229407Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:26:59Z"} 2026-03-20T11:26:59.490 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:26:59.600 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:59.600 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:59.600 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:59.635 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:59.636 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:59.706 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:26:59.716 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006019.557783_44.1","last_update":"2026-03-20T11:26:59.557783Z"},{"marker":"1_1774005907.643115_44.1","last_update":"2026-03-20T11:25:07.643115Z"},{"marker":"1_1774006019.555147_79.1","last_update":"2026-03-20T11:26:59.555147Z"}] 2026-03-20T11:26:59.716 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:26:59.716 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:59.716 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:26:59.717 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:26:59.795 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:26:59.795 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:26:59.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:59.857+0000 7f88e7c10900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:59.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:26:59.858+0000 7f88e7c10900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:26:59.866 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006019.557783_44.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:26:59.557783Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774005907.643115_44.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:25:07.643115Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006019.555147_79.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:26:59.555147Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:26:59Z"} 2026-03-20T11:26:59.866 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:26:59.866 INFO:rgw_multi.tests:Checking zone=a1 for deleted buckets 2026-03-20T11:26:59.868 CRITICAL:rgw_multi.tests:zone contains bucket dipepu-5 2026-03-20T11:26:59.868 ERROR:rgw_multi.tests:Zone a1 still has buckets: ['dipepu-5', 'dipepu-6'] 2026-03-20T11:26:59.869 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_remove ... FAIL 2026-03-20T11:26:59.941 INFO:rgw_multi.tests:create bucket zone=a1 name=dipepu-7 2026-03-20T11:26:59.954 INFO:rgw_multi.tests:create bucket zone=a2 name=dipepu-8 2026-03-20T11:27:00.086 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:27:00.087 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:27:00.087 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:27:00.128 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:27:00.128 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:27:00.192 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:27:00.201 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006019.966136_46.1","last_update":"2026-03-20T11:26:59.966136Z"},{"marker":"1_1774005907.643115_44.1","last_update":"2026-03-20T11:25:07.643115Z"},{"marker":"1_1774006019.968691_81.1","last_update":"2026-03-20T11:26:59.968691Z"}] 2026-03-20T11:27:00.202 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:27:00.202 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:27:00.202 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:27:00.202 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:27:00.278 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:27:00.278 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:27:00.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:00.344+0000 7f5644786900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:27:00.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:00.345+0000 7f5644786900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:27:00.353 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006019.966136_46.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:26:59.966136Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774005907.643115_44.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:25:07.643115Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006019.968691_81.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:26:59.968691Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:27:00Z"} 2026-03-20T11:27:00.353 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:27:00.353 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-7', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:27:00.354 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-7', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:27:00.354 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-7 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:27:00.433 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:27:00.433 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:27:00.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:00.501+0000 7f0310410900 1 waiting to reach incremental sync.. 2026-03-20T11:27:05.501 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:05.502+0000 7f0310410900 1 bucket sync caught up with source: 2026-03-20T11:27:05.501 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, 00000000002.6.6, , 00000000001.10.6, , 00000000001.6.6, 00000000002.6.6, 00000000001.6.6, 00000000001.12.6, 00000000001.4.6, ] 2026-03-20T11:27:05.501 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, 00000000002.6.6, , 00000000001.10.6, , 00000000001.6.6, 00000000002.6.6, 00000000001.6.6, 00000000001.12.6, 00000000001.4.6, ] 2026-03-20T11:27:05.501 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:05.502+0000 7f0310410900 0 bucket checkpoint complete 2026-03-20T11:27:05.509 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-7 zones={a2, a1} 2026-03-20T11:27:05.524 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:27:05.524 DEBUG:tasks.rgw_multi.zone_rados:o=&, v=null 2026-03-20T11:27:05.524 DEBUG:tasks.rgw_multi.zone_rados:o=., v=null 2026-03-20T11:27:05.524 DEBUG:tasks.rgw_multi.zone_rados:o=.., v=null 2026-03-20T11:27:05.524 DEBUG:tasks.rgw_multi.zone_rados:o=..., v=null 2026-03-20T11:27:05.524 DEBUG:tasks.rgw_multi.zone_rados:o=.o, v=null 2026-03-20T11:27:05.524 DEBUG:tasks.rgw_multi.zone_rados:o=.o., v=null 2026-03-20T11:27:05.524 DEBUG:tasks.rgw_multi.zone_rados:o=:, v=null 2026-03-20T11:27:05.524 DEBUG:tasks.rgw_multi.zone_rados:o=_myobj, v=null 2026-03-20T11:27:05.524 DEBUG:tasks.rgw_multi.zone_rados:o=myobj, v=null 2026-03-20T11:27:05.527 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:27:05.527 DEBUG:tasks.rgw_multi.zone_rados:o=&, v=null 2026-03-20T11:27:05.527 DEBUG:tasks.rgw_multi.zone_rados:o=., v=null 2026-03-20T11:27:05.527 DEBUG:tasks.rgw_multi.zone_rados:o=.., v=null 2026-03-20T11:27:05.527 DEBUG:tasks.rgw_multi.zone_rados:o=..., v=null 2026-03-20T11:27:05.527 DEBUG:tasks.rgw_multi.zone_rados:o=.o, v=null 2026-03-20T11:27:05.527 DEBUG:tasks.rgw_multi.zone_rados:o=.o., v=null 2026-03-20T11:27:05.527 DEBUG:tasks.rgw_multi.zone_rados:o=:, v=null 2026-03-20T11:27:05.527 DEBUG:tasks.rgw_multi.zone_rados:o=_myobj, v=null 2026-03-20T11:27:05.527 DEBUG:tasks.rgw_multi.zone_rados:o=myobj, v=null 2026-03-20T11:27:05.527 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2026-03-20T11:27:05.588 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=. 2026-03-20T11:27:05.647 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=.. 2026-03-20T11:27:05.703 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=... 2026-03-20T11:27:05.760 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=.o 2026-03-20T11:27:05.815 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=.o. 2026-03-20T11:27:05.875 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2026-03-20T11:27:05.933 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2026-03-20T11:27:05.992 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2026-03-20T11:27:06.047 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-7 zones={a2, a1} 2026-03-20T11:27:06.047 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-8', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:27:06.047 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-8', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:27:06.047 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-8 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:27:06.088 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:27:06.088 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:27:06.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:06.159+0000 7fad803cb900 1 bucket sync caught up with source: 2026-03-20T11:27:06.158 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, 00000000002.13.6, , 00000000001.5.6, , 00000000001.2.6, 00000000002.13.6, 00000000001.11.6, 00000000001.4.6, 00000000001.12.6, ] 2026-03-20T11:27:06.158 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, 00000000002.13.6, , 00000000001.5.6, , 00000000001.2.6, 00000000002.13.6, 00000000001.11.6, 00000000001.4.6, 00000000001.12.6, ] 2026-03-20T11:27:06.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:06.159+0000 7fad803cb900 0 bucket checkpoint complete 2026-03-20T11:27:06.167 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-8 zones={a1, a2} 2026-03-20T11:27:06.174 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:27:06.174 DEBUG:tasks.rgw_multi.zone_rados:o=&, v=null 2026-03-20T11:27:06.174 DEBUG:tasks.rgw_multi.zone_rados:o=., v=null 2026-03-20T11:27:06.174 DEBUG:tasks.rgw_multi.zone_rados:o=.., v=null 2026-03-20T11:27:06.174 DEBUG:tasks.rgw_multi.zone_rados:o=..., v=null 2026-03-20T11:27:06.174 DEBUG:tasks.rgw_multi.zone_rados:o=.o, v=null 2026-03-20T11:27:06.174 DEBUG:tasks.rgw_multi.zone_rados:o=.o., v=null 2026-03-20T11:27:06.174 DEBUG:tasks.rgw_multi.zone_rados:o=:, v=null 2026-03-20T11:27:06.174 DEBUG:tasks.rgw_multi.zone_rados:o=_myobj, v=null 2026-03-20T11:27:06.174 DEBUG:tasks.rgw_multi.zone_rados:o=myobj, v=null 2026-03-20T11:27:06.177 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:27:06.177 DEBUG:tasks.rgw_multi.zone_rados:o=&, v=null 2026-03-20T11:27:06.177 DEBUG:tasks.rgw_multi.zone_rados:o=., v=null 2026-03-20T11:27:06.177 DEBUG:tasks.rgw_multi.zone_rados:o=.., v=null 2026-03-20T11:27:06.177 DEBUG:tasks.rgw_multi.zone_rados:o=..., v=null 2026-03-20T11:27:06.177 DEBUG:tasks.rgw_multi.zone_rados:o=.o, v=null 2026-03-20T11:27:06.177 DEBUG:tasks.rgw_multi.zone_rados:o=.o., v=null 2026-03-20T11:27:06.177 DEBUG:tasks.rgw_multi.zone_rados:o=:, v=null 2026-03-20T11:27:06.177 DEBUG:tasks.rgw_multi.zone_rados:o=_myobj, v=null 2026-03-20T11:27:06.177 DEBUG:tasks.rgw_multi.zone_rados:o=myobj, v=null 2026-03-20T11:27:06.177 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2026-03-20T11:27:06.232 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=. 2026-03-20T11:27:06.288 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=.. 2026-03-20T11:27:06.403 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=... 2026-03-20T11:27:06.460 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=.o 2026-03-20T11:27:06.517 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=.o. 2026-03-20T11:27:06.572 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2026-03-20T11:27:06.631 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2026-03-20T11:27:06.695 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2026-03-20T11:27:06.757 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-8 zones={a1, a2} 2026-03-20T11:27:06.760 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_object_sync ... ok 2026-03-20T11:27:06.806 INFO:rgw_multi.tests:create bucket zone=a1 name=dipepu-9 2026-03-20T11:27:06.822 INFO:rgw_multi.tests:create bucket zone=a2 name=dipepu-10 2026-03-20T11:27:06.860 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:27:06.860 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:27:06.860 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:27:06.900 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:27:06.900 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:27:06.969 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:27:06.978 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774006026.836914_60.1","last_update":"2026-03-20T11:27:06.836914Z"},{"marker":"1_1774006026.822813_82.1","last_update":"2026-03-20T11:27:06.822813Z"}] 2026-03-20T11:27:06.978 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:27:06.978 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:27:06.978 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:27:06.979 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:27:07.055 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:27:07.055 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:27:07.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:07.123+0000 7f365bd1f900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:27:07.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:07.124+0000 7f365bd1f900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:27:07.133 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006026.836914_60.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.836914Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006026.822813_82.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.822813Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:27:07Z"} 2026-03-20T11:27:07.133 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:27:07.133 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-9', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:27:07.133 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-9', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:27:07.133 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-9 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:27:07.214 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:27:07.214 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:27:07.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:07.285+0000 7f95f6728900 1 waiting to reach incremental sync.. 2026-03-20T11:27:12.286 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:12.287+0000 7f95f6728900 1 bucket sync caught up with source: 2026-03-20T11:27:12.286 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, 00000000001.11.6, , , , , , , , , ] 2026-03-20T11:27:12.286 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, 00000000001.11.6, , , , , , , , , ] 2026-03-20T11:27:12.286 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:12.287+0000 7f95f6728900 0 bucket checkpoint complete 2026-03-20T11:27:12.296 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-9 zones={a2, a1} 2026-03-20T11:27:12.304 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:27:12.304 DEBUG:tasks.rgw_multi.zone_rados:o=myobj, v=null 2026-03-20T11:27:12.307 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:27:12.307 DEBUG:tasks.rgw_multi.zone_rados:o=myobj, v=null 2026-03-20T11:27:12.307 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2026-03-20T11:27:12.367 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-9 zones={a2, a1} 2026-03-20T11:27:12.368 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-10', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:27:12.368 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-10', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:27:12.368 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-10 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:27:12.407 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:27:12.407 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:27:12.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:12.471+0000 7fe9c3c10900 1 bucket sync caught up with source: 2026-03-20T11:27:12.469 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, 00000000001.5.6, , , , , , , , , ] 2026-03-20T11:27:12.469 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, 00000000001.5.6, , , , , , , , , ] 2026-03-20T11:27:12.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:12.471+0000 7fe9c3c10900 0 bucket checkpoint complete 2026-03-20T11:27:12.479 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-10 zones={a1, a2} 2026-03-20T11:27:12.485 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:27:12.485 DEBUG:tasks.rgw_multi.zone_rados:o=myobj, v=null 2026-03-20T11:27:12.487 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:27:12.487 DEBUG:tasks.rgw_multi.zone_rados:o=myobj, v=null 2026-03-20T11:27:12.487 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2026-03-20T11:27:12.543 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-10 zones={a1, a2} 2026-03-20T11:27:12.549 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-9', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:27:12.549 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-9', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:27:12.549 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-9 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:27:12.590 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:27:12.590 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:27:12.688 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:12.689+0000 7f32a43cd900 1 waiting for incremental sync to catch up: 2026-03-20T11:27:12.688 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, 00000000001.11.6, , , , , , , , , ] 2026-03-20T11:27:12.688 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, 00000000002.13.6, , , , , , , , , ] 2026-03-20T11:27:17.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:17.690+0000 7f32a43cd900 1 waiting for incremental sync to catch up: 2026-03-20T11:27:17.689 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, 00000000001.11.6, , , , , , , , , ] 2026-03-20T11:27:17.689 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, 00000000002.13.6, , , , , , , , , ] 2026-03-20T11:27:22.690 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:22.691+0000 7f32a43cd900 1 bucket sync caught up with source: 2026-03-20T11:27:22.690 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, 00000000002.13.6, , , , , , , , , ] 2026-03-20T11:27:22.690 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, 00000000002.13.6, , , , , , , , , ] 2026-03-20T11:27:22.690 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:22.691+0000 7f32a43cd900 0 bucket checkpoint complete 2026-03-20T11:27:22.699 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-9 zones={a2, a1} 2026-03-20T11:27:22.704 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:27:22.706 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:27:22.706 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-9 zones={a2, a1} 2026-03-20T11:27:22.711 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-10', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:27:22.711 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-10', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:27:22.711 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-10 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:27:22.779 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:27:22.779 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:27:22.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:22.853+0000 7f590852d900 1 waiting for incremental sync to catch up: 2026-03-20T11:27:22.851 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, 00000000001.5.6, , , , , , , , , ] 2026-03-20T11:27:22.851 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, 00000000002.9.6, , , , , , , , , ] 2026-03-20T11:27:27.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:27.854+0000 7f590852d900 1 waiting for incremental sync to catch up: 2026-03-20T11:27:27.852 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, 00000000001.5.6, , , , , , , , , ] 2026-03-20T11:27:27.852 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, 00000000002.9.6, , , , , , , , , ] 2026-03-20T11:27:32.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:32.854+0000 7f590852d900 1 waiting for incremental sync to catch up: 2026-03-20T11:27:32.853 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, 00000000001.5.6, , , , , , , , , ] 2026-03-20T11:27:32.853 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, 00000000002.9.6, , , , , , , , , ] 2026-03-20T11:27:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:37.856+0000 7f590852d900 1 waiting for incremental sync to catch up: 2026-03-20T11:27:37.854 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, 00000000001.5.6, , , , , , , , , ] 2026-03-20T11:27:37.854 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, 00000000002.9.6, , , , , , , , , ] 2026-03-20T11:27:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:42.857+0000 7f590852d900 1 bucket sync caught up with source: 2026-03-20T11:27:42.856 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, 00000000002.9.6, , , , , , , , , ] 2026-03-20T11:27:42.856 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, 00000000002.9.6, , , , , , , , , ] 2026-03-20T11:27:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:42.857+0000 7f590852d900 0 bucket checkpoint complete 2026-03-20T11:27:42.865 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-10 zones={a1, a2} 2026-03-20T11:27:42.871 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:27:42.874 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:27:42.874 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-10 zones={a1, a2} 2026-03-20T11:27:42.876 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_object_delete ... ok 2026-03-20T11:27:42.969 INFO:rgw_multi.tests:create bucket zone=a1 name=dipepu-11 2026-03-20T11:27:42.982 INFO:rgw_multi.tests:create bucket zone=a2 name=dipepu-12 2026-03-20T11:27:43.455 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:27:43.455 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:27:43.455 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:27:43.492 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:27:43.492 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:27:43.555 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:27:43.564 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774006062.996890_66.1","last_update":"2026-03-20T11:27:42.996890Z"},{"marker":"1_1774006026.822813_82.1","last_update":"2026-03-20T11:27:06.822813Z"}] 2026-03-20T11:27:43.565 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:27:43.565 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:27:43.565 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:27:43.565 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:27:43.640 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:27:43.640 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:27:43.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:43.711+0000 7f7e3d92d900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:27:43.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:43.712+0000 7f7e3d92d900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:27:43.720 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006062.996890_66.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:42.996890Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006026.822813_82.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.822813Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:27:43Z"} 2026-03-20T11:27:43.720 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:27:43.720 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-11', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:27:43.720 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-11', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:27:43.720 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-11 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:27:43.798 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:27:43.798 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:27:43.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:43.871+0000 7f4df8218900 1 waiting to reach incremental sync.. 2026-03-20T11:27:48.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:48.873+0000 7f4df8218900 1 bucket sync caught up with source: 2026-03-20T11:27:48.872 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000007.30.6, 00000000004.22.6, 00000000004.26.6, 00000000004.16.6, 00000000006.16.6, 00000000005.21.6, 00000000004.13.6, 00000000005.17.6, 00000000002.15.6, 00000000004.14.6, 00000000004.11.6] 2026-03-20T11:27:48.872 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000007.30.6, 00000000004.22.6, 00000000004.26.6, 00000000004.16.6, 00000000006.16.6, 00000000005.21.6, 00000000004.13.6, 00000000005.17.6, 00000000002.15.6, 00000000004.14.6, 00000000004.11.6] 2026-03-20T11:27:48.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:48.873+0000 7f4df8218900 0 bucket checkpoint complete 2026-03-20T11:27:48.881 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-11 zones={a2, a1} 2026-03-20T11:27:48.892 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:27:48.892 DEBUG:tasks.rgw_multi.zone_rados:o=obj1, v=null 2026-03-20T11:27:48.892 DEBUG:tasks.rgw_multi.zone_rados:o=obj10, v=null 2026-03-20T11:27:48.892 DEBUG:tasks.rgw_multi.zone_rados:o=obj11, v=null 2026-03-20T11:27:48.892 DEBUG:tasks.rgw_multi.zone_rados:o=obj12, v=null 2026-03-20T11:27:48.892 DEBUG:tasks.rgw_multi.zone_rados:o=obj13, v=null 2026-03-20T11:27:48.892 DEBUG:tasks.rgw_multi.zone_rados:o=obj14, v=null 2026-03-20T11:27:48.892 DEBUG:tasks.rgw_multi.zone_rados:o=obj15, v=null 2026-03-20T11:27:48.892 DEBUG:tasks.rgw_multi.zone_rados:o=obj16, v=null 2026-03-20T11:27:48.892 DEBUG:tasks.rgw_multi.zone_rados:o=obj17, v=null 2026-03-20T11:27:48.892 DEBUG:tasks.rgw_multi.zone_rados:o=obj18, v=null 2026-03-20T11:27:48.892 DEBUG:tasks.rgw_multi.zone_rados:o=obj19, v=null 2026-03-20T11:27:48.892 DEBUG:tasks.rgw_multi.zone_rados:o=obj2, v=null 2026-03-20T11:27:48.892 DEBUG:tasks.rgw_multi.zone_rados:o=obj20, v=null 2026-03-20T11:27:48.892 DEBUG:tasks.rgw_multi.zone_rados:o=obj21, v=null 2026-03-20T11:27:48.892 DEBUG:tasks.rgw_multi.zone_rados:o=obj22, v=null 2026-03-20T11:27:48.892 DEBUG:tasks.rgw_multi.zone_rados:o=obj23, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj24, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj25, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj26, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj27, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj28, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj29, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj3, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj30, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj31, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj32, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj33, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj34, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj35, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj36, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj37, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj38, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj39, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj4, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj40, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj41, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj42, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj43, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj44, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj45, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj46, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj47, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj48, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj49, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj5, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj6, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj7, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj8, v=null 2026-03-20T11:27:48.893 DEBUG:tasks.rgw_multi.zone_rados:o=obj9, v=null 2026-03-20T11:27:48.899 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:27:48.899 DEBUG:tasks.rgw_multi.zone_rados:o=obj1, v=null 2026-03-20T11:27:48.899 DEBUG:tasks.rgw_multi.zone_rados:o=obj10, v=null 2026-03-20T11:27:48.899 DEBUG:tasks.rgw_multi.zone_rados:o=obj11, v=null 2026-03-20T11:27:48.899 DEBUG:tasks.rgw_multi.zone_rados:o=obj12, v=null 2026-03-20T11:27:48.899 DEBUG:tasks.rgw_multi.zone_rados:o=obj13, v=null 2026-03-20T11:27:48.899 DEBUG:tasks.rgw_multi.zone_rados:o=obj14, v=null 2026-03-20T11:27:48.899 DEBUG:tasks.rgw_multi.zone_rados:o=obj15, v=null 2026-03-20T11:27:48.899 DEBUG:tasks.rgw_multi.zone_rados:o=obj16, v=null 2026-03-20T11:27:48.899 DEBUG:tasks.rgw_multi.zone_rados:o=obj17, v=null 2026-03-20T11:27:48.899 DEBUG:tasks.rgw_multi.zone_rados:o=obj18, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj19, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj2, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj20, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj21, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj22, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj23, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj24, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj25, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj26, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj27, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj28, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj29, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj3, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj30, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj31, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj32, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj33, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj34, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj35, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj36, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj37, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj38, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj39, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj4, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj40, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj41, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj42, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj43, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj44, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj45, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj46, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj47, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj48, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj49, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj5, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj6, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj7, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj8, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:o=obj9, v=null 2026-03-20T11:27:48.900 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2026-03-20T11:27:48.956 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj10 2026-03-20T11:27:49.014 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2026-03-20T11:27:49.071 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj12 2026-03-20T11:27:49.131 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj13 2026-03-20T11:27:49.186 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj14 2026-03-20T11:27:49.242 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj15 2026-03-20T11:27:49.298 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj16 2026-03-20T11:27:49.354 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2026-03-20T11:27:49.410 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj18 2026-03-20T11:27:49.464 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2026-03-20T11:27:49.521 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2026-03-20T11:27:49.607 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj20 2026-03-20T11:27:49.666 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj21 2026-03-20T11:27:49.722 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2026-03-20T11:27:49.778 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2026-03-20T11:27:49.834 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj24 2026-03-20T11:27:49.890 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj25 2026-03-20T11:27:49.942 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj26 2026-03-20T11:27:49.994 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj27 2026-03-20T11:27:50.050 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj28 2026-03-20T11:27:50.116 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2026-03-20T11:27:50.170 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2026-03-20T11:27:50.227 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj30 2026-03-20T11:27:50.282 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj31 2026-03-20T11:27:50.334 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj32 2026-03-20T11:27:50.386 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2026-03-20T11:27:50.444 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2026-03-20T11:27:50.500 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj35 2026-03-20T11:27:50.555 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj36 2026-03-20T11:27:50.612 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj37 2026-03-20T11:27:50.667 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2026-03-20T11:27:50.724 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj39 2026-03-20T11:27:50.779 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2026-03-20T11:27:50.837 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj40 2026-03-20T11:27:50.891 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj41 2026-03-20T11:27:50.947 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj42 2026-03-20T11:27:51.011 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj43 2026-03-20T11:27:51.071 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2026-03-20T11:27:51.132 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj45 2026-03-20T11:27:51.187 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2026-03-20T11:27:51.247 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj47 2026-03-20T11:27:51.309 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj48 2026-03-20T11:27:51.368 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj49 2026-03-20T11:27:51.428 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj5 2026-03-20T11:27:51.488 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj6 2026-03-20T11:27:51.547 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj7 2026-03-20T11:27:51.602 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj8 2026-03-20T11:27:51.658 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj9 2026-03-20T11:27:51.711 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-11 zones={a2, a1} 2026-03-20T11:27:51.711 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-12', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:27:51.711 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-12', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:27:51.711 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-12 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:27:51.749 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:27:51.750 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:27:51.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:51.818+0000 7f6a4c730900 1 bucket sync caught up with source: 2026-03-20T11:27:51.816 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000007.25.6, 00000000004.27.6, 00000000004.19.6, 00000000004.15.6, 00000000006.18.6, 00000000005.15.6, 00000000004.20.6, 00000000005.19.6, 00000000002.22.6, 00000000004.20.6, 00000000004.18.6] 2026-03-20T11:27:51.816 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000007.25.6, 00000000004.27.6, 00000000004.19.6, 00000000004.15.6, 00000000006.18.6, 00000000005.15.6, 00000000004.20.6, 00000000005.19.6, 00000000002.22.6, 00000000004.20.6, 00000000004.18.6] 2026-03-20T11:27:51.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:51.818+0000 7f6a4c730900 0 bucket checkpoint complete 2026-03-20T11:27:51.837 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-12 zones={a1, a2} 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj1, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj10, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj11, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj12, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj13, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj14, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj15, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj16, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj17, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj18, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj19, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj2, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj20, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj21, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj22, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj23, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj24, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj25, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj26, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj27, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj28, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj29, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj3, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj30, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj31, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj32, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj33, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj34, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj35, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj36, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj37, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj38, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj39, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj4, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj40, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj41, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj42, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj43, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj44, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj45, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj46, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj47, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj48, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj49, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj5, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj6, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj7, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj8, v=null 2026-03-20T11:27:51.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj9, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj1, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj10, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj11, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj12, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj13, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj14, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj15, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj16, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj17, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj18, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj19, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj2, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj20, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj21, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj22, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj23, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj24, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj25, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj26, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj27, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj28, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj29, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj3, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj30, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj31, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj32, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj33, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj34, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj35, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj36, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj37, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj38, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj39, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj4, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj40, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj41, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj42, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj43, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj44, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj45, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj46, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj47, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj48, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj49, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj5, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj6, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj7, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj8, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj9, v=null 2026-03-20T11:27:51.853 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2026-03-20T11:27:51.910 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj10 2026-03-20T11:27:51.965 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj11 2026-03-20T11:27:52.022 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj12 2026-03-20T11:27:52.078 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj13 2026-03-20T11:27:52.134 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj14 2026-03-20T11:27:52.185 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj15 2026-03-20T11:27:52.242 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj16 2026-03-20T11:27:52.298 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj17 2026-03-20T11:27:52.349 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj18 2026-03-20T11:27:52.405 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj19 2026-03-20T11:27:52.462 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2026-03-20T11:27:52.513 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj20 2026-03-20T11:27:52.570 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj21 2026-03-20T11:27:52.623 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj22 2026-03-20T11:27:52.677 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj23 2026-03-20T11:27:52.729 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj24 2026-03-20T11:27:52.782 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj25 2026-03-20T11:27:52.838 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj26 2026-03-20T11:27:52.893 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj27 2026-03-20T11:27:52.949 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj28 2026-03-20T11:27:53.001 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj29 2026-03-20T11:27:53.058 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2026-03-20T11:27:53.109 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj30 2026-03-20T11:27:53.162 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj31 2026-03-20T11:27:53.213 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj32 2026-03-20T11:27:53.266 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj33 2026-03-20T11:27:53.321 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj34 2026-03-20T11:27:53.373 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj35 2026-03-20T11:27:53.425 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj36 2026-03-20T11:27:53.478 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj37 2026-03-20T11:27:53.530 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj38 2026-03-20T11:27:53.581 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj39 2026-03-20T11:27:53.634 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2026-03-20T11:27:53.741 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj40 2026-03-20T11:27:53.798 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj41 2026-03-20T11:27:53.853 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj42 2026-03-20T11:27:53.910 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj43 2026-03-20T11:27:53.961 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj44 2026-03-20T11:27:54.017 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj45 2026-03-20T11:27:54.073 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj46 2026-03-20T11:27:54.131 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj47 2026-03-20T11:27:54.185 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj48 2026-03-20T11:27:54.241 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj49 2026-03-20T11:27:54.297 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj5 2026-03-20T11:27:54.354 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj6 2026-03-20T11:27:54.405 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj7 2026-03-20T11:27:54.457 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj8 2026-03-20T11:27:54.509 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj9 2026-03-20T11:27:54.562 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-12 zones={a1, a2} 2026-03-20T11:27:54.602 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-11', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:27:54.602 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-11', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:27:54.602 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-11 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:27:54.643 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:27:54.643 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:27:54.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:54.712+0000 7fa3c81d0900 1 waiting for incremental sync to catch up: 2026-03-20T11:27:54.711 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000007.30.6, 00000000004.22.6, 00000000004.26.6, 00000000004.16.6, 00000000006.16.6, 00000000005.21.6, 00000000004.13.6, 00000000005.17.6, 00000000002.15.6, 00000000004.14.6, 00000000004.11.6] 2026-03-20T11:27:54.711 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000014.52.6, 00000000008.34.6, 00000000008.50.6, 00000000008.32.6, 00000000012.28.6, 00000000010.31.6, 00000000008.21.6, 00000000010.27.6, 00000000004.27.6, 00000000008.22.6, 00000000008.19.6] 2026-03-20T11:27:59.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:27:59.713+0000 7fa3c81d0900 1 waiting for incremental sync to catch up: 2026-03-20T11:27:59.712 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000007.30.6, 00000000004.22.6, 00000000004.26.6, 00000000004.16.6, 00000000006.16.6, 00000000005.21.6, 00000000004.13.6, 00000000005.17.6, 00000000002.15.6, 00000000004.14.6, 00000000004.11.6] 2026-03-20T11:27:59.712 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000014.52.6, 00000000008.34.6, 00000000008.50.6, 00000000008.32.6, 00000000012.28.6, 00000000010.31.6, 00000000008.21.6, 00000000010.27.6, 00000000004.27.6, 00000000008.22.6, 00000000008.19.6] 2026-03-20T11:28:04.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:04.714+0000 7fa3c81d0900 1 bucket sync caught up with source: 2026-03-20T11:28:04.713 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000014.52.6, 00000000008.34.6, 00000000008.50.6, 00000000008.32.6, 00000000012.28.6, 00000000010.31.6, 00000000008.21.6, 00000000010.27.6, 00000000004.27.6, 00000000008.22.6, 00000000008.19.6] 2026-03-20T11:28:04.713 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000014.52.6, 00000000008.34.6, 00000000008.50.6, 00000000008.32.6, 00000000012.28.6, 00000000010.31.6, 00000000008.21.6, 00000000010.27.6, 00000000004.27.6, 00000000008.22.6, 00000000008.19.6] 2026-03-20T11:28:04.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:04.714+0000 7fa3c81d0900 0 bucket checkpoint complete 2026-03-20T11:28:04.723 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-11 zones={a2, a1} 2026-03-20T11:28:04.729 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:28:04.731 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:28:04.731 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-11 zones={a2, a1} 2026-03-20T11:28:04.807 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-12', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:28:04.807 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-12', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:28:04.807 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-12 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:28:04.847 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:28:04.848 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:28:04.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:04.930+0000 7f6356f4b900 1 waiting for incremental sync to catch up: 2026-03-20T11:28:04.929 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000007.25.6, 00000000004.27.6, 00000000004.19.6, 00000000004.15.6, 00000000006.18.6, 00000000005.15.6, 00000000004.20.6, 00000000005.19.6, 00000000002.22.6, 00000000004.20.6, 00000000004.18.6] 2026-03-20T11:28:04.929 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000014.49.6, 00000000008.48.6, 00000000008.43.6, 00000000008.23.6, 00000000012.30.6, 00000000010.25.6, 00000000008.36.6, 00000000010.29.6, 00000000004.42.6, 00000000008.28.6, 00000000008.26.6] 2026-03-20T11:28:09.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:09.931+0000 7f6356f4b900 1 waiting for incremental sync to catch up: 2026-03-20T11:28:09.930 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000007.25.6, 00000000004.27.6, 00000000004.19.6, 00000000004.15.6, 00000000006.18.6, 00000000005.15.6, 00000000004.20.6, 00000000005.19.6, 00000000002.22.6, 00000000004.20.6, 00000000004.18.6] 2026-03-20T11:28:09.930 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000014.49.6, 00000000008.48.6, 00000000008.43.6, 00000000008.23.6, 00000000012.30.6, 00000000010.25.6, 00000000008.36.6, 00000000010.29.6, 00000000004.42.6, 00000000008.28.6, 00000000008.26.6] 2026-03-20T11:28:14.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:14.932+0000 7f6356f4b900 1 waiting for incremental sync to catch up: 2026-03-20T11:28:14.931 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000007.25.6, 00000000004.27.6, 00000000004.19.6, 00000000004.15.6, 00000000006.18.6, 00000000005.15.6, 00000000004.20.6, 00000000005.19.6, 00000000002.22.6, 00000000004.20.6, 00000000004.18.6] 2026-03-20T11:28:14.931 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000014.49.6, 00000000008.48.6, 00000000008.43.6, 00000000008.23.6, 00000000012.30.6, 00000000010.25.6, 00000000008.36.6, 00000000010.29.6, 00000000004.42.6, 00000000008.28.6, 00000000008.26.6] 2026-03-20T11:28:19.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:19.933+0000 7f6356f4b900 1 waiting for incremental sync to catch up: 2026-03-20T11:28:19.932 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000007.25.6, 00000000004.27.6, 00000000004.19.6, 00000000004.15.6, 00000000006.18.6, 00000000005.15.6, 00000000004.20.6, 00000000005.19.6, 00000000002.22.6, 00000000004.20.6, 00000000004.18.6] 2026-03-20T11:28:19.932 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000014.49.6, 00000000008.48.6, 00000000008.43.6, 00000000008.23.6, 00000000012.30.6, 00000000010.25.6, 00000000008.36.6, 00000000010.29.6, 00000000004.42.6, 00000000008.28.6, 00000000008.26.6] 2026-03-20T11:28:24.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:24.934+0000 7f6356f4b900 1 bucket sync caught up with source: 2026-03-20T11:28:24.933 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000014.49.6, 00000000008.48.6, 00000000008.43.6, 00000000008.23.6, 00000000012.30.6, 00000000010.25.6, 00000000008.36.6, 00000000010.29.6, 00000000004.42.6, 00000000008.28.6, 00000000008.26.6] 2026-03-20T11:28:24.933 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000014.49.6, 00000000008.48.6, 00000000008.43.6, 00000000008.23.6, 00000000012.30.6, 00000000010.25.6, 00000000008.36.6, 00000000010.29.6, 00000000004.42.6, 00000000008.28.6, 00000000008.26.6] 2026-03-20T11:28:24.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:24.934+0000 7f6356f4b900 0 bucket checkpoint complete 2026-03-20T11:28:24.943 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-12 zones={a1, a2} 2026-03-20T11:28:24.950 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:28:24.953 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:28:24.953 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-12 zones={a1, a2} 2026-03-20T11:28:24.955 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_multi_object_delete ... ok 2026-03-20T11:28:25.086 INFO:rgw_multi.tests:create bucket zone=a1 name=dipepu-13 2026-03-20T11:28:25.101 INFO:rgw_multi.tests:create bucket zone=a2 name=dipepu-14 2026-03-20T11:28:25.139 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:28:25.139 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:28:25.139 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:28:25.176 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:28:25.176 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:28:25.257 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:28:25.268 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774006105.138312_72.1","last_update":"2026-03-20T11:28:25.138312Z"},{"marker":"1_1774006026.822813_82.1","last_update":"2026-03-20T11:27:06.822813Z"}] 2026-03-20T11:28:25.268 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:28:25.268 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:28:25.269 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:28:25.269 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:28:25.351 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:28:25.352 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:28:25.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:25.430+0000 7fe957e12900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:28:25.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:25.431+0000 7fe957e12900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:28:25.441 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006105.138312_72.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:28:25.138312Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006026.822813_82.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.822813Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:28:25Z"} 2026-03-20T11:28:25.441 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:28:25.457 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-13', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:28:25.457 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-13', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:28:25.457 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-13 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:28:25.505 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:28:25.505 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:28:25.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:25.582+0000 7f182c11f900 1 waiting to reach incremental sync.. 2026-03-20T11:28:30.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:30.584+0000 7f182c11f900 1 bucket sync caught up with source: 2026-03-20T11:28:30.582 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000002.27.11, , , , , ] 2026-03-20T11:28:30.582 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000002.27.11, , , , , ] 2026-03-20T11:28:30.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:30.584+0000 7f182c11f900 0 bucket checkpoint complete 2026-03-20T11:28:30.606 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-14', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:28:30.606 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-14', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:28:30.607 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-14 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:28:30.648 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:28:30.648 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:28:30.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:30.732+0000 7fd802e1a900 1 waiting to reach incremental sync.. 2026-03-20T11:28:35.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:35.734+0000 7fd802e1a900 1 bucket sync caught up with source: 2026-03-20T11:28:35.732 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000002.34.11, , , , , ] 2026-03-20T11:28:35.732 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000002.34.11, , , , , ] 2026-03-20T11:28:35.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:35.734+0000 7fd802e1a900 0 bucket checkpoint complete 2026-03-20T11:28:35.950 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-13', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:28:35.950 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-13', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:28:35.950 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-13 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:28:35.995 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:28:35.995 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:28:36.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:36.081+0000 7fe524d25900 1 waiting for incremental sync to catch up: 2026-03-20T11:28:36.079 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000002.27.11, , , , , ] 2026-03-20T11:28:36.079 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000003.29.10, , , , 00000000010.34.6, ] 2026-03-20T11:28:41.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:41.082+0000 7fe524d25900 1 waiting for incremental sync to catch up: 2026-03-20T11:28:41.081 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000002.27.11, , , , 00000000010.34.6, ] 2026-03-20T11:28:41.081 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000003.29.10, , , , 00000000010.34.6, ] 2026-03-20T11:28:46.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:46.083+0000 7fe524d25900 1 waiting for incremental sync to catch up: 2026-03-20T11:28:46.082 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000002.27.11, , , 00000000002.8.11, 00000000010.34.6, ] 2026-03-20T11:28:46.082 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000003.29.10, , , , 00000000010.34.6, ] 2026-03-20T11:28:51.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:51.084+0000 7fe524d25900 1 bucket sync caught up with source: 2026-03-20T11:28:51.083 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000003.29.10, , , 00000000002.8.11, 00000000010.34.6, ] 2026-03-20T11:28:51.083 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000003.29.10, , , , 00000000010.34.6, ] 2026-03-20T11:28:51.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:51.084+0000 7fe524d25900 0 bucket checkpoint complete 2026-03-20T11:28:51.093 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-13', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:28:51.093 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-13', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:28:51.093 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-13 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:28:51.137 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:28:51.137 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:28:51.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:51.218+0000 7f5b63586900 1 bucket sync caught up with source: 2026-03-20T11:28:51.216 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000002.36.11, , , 00000000010.22.6, 00000000002.17.11, ] 2026-03-20T11:28:51.216 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000002.36.11, , , 00000000010.22.6, 00000000002.17.11, ] 2026-03-20T11:28:51.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:51.218+0000 7f5b63586900 0 bucket checkpoint complete 2026-03-20T11:28:51.226 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-13 zones={a2, a1} 2026-03-20T11:28:51.234 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:28:51.234 DEBUG:tasks.rgw_multi.zone_rados:o=dummy, v=Xi2I5D1jFJoS21REWf-2D6eyzZVpgFm 2026-03-20T11:28:51.234 DEBUG:tasks.rgw_multi.zone_rados:o=obj-a1, v=13IwQ-odIOdimI1E5hsY2ZSkETyWvuV 2026-03-20T11:28:51.234 DEBUG:tasks.rgw_multi.zone_rados:o=obj-a2, v=jMNSPaa.HpmlL9uZS79AYqRdglaSpdK 2026-03-20T11:28:51.237 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:28:51.237 DEBUG:tasks.rgw_multi.zone_rados:o=dummy, v=Xi2I5D1jFJoS21REWf-2D6eyzZVpgFm 2026-03-20T11:28:51.237 DEBUG:tasks.rgw_multi.zone_rados:o=obj-a1, v=13IwQ-odIOdimI1E5hsY2ZSkETyWvuV 2026-03-20T11:28:51.237 DEBUG:tasks.rgw_multi.zone_rados:o=obj-a2, v=jMNSPaa.HpmlL9uZS79AYqRdglaSpdK 2026-03-20T11:28:51.237 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2026-03-20T11:28:51.252 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-a1 2026-03-20T11:28:51.328 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-a2 2026-03-20T11:28:51.391 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-13 zones={a2, a1} 2026-03-20T11:28:51.392 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-14', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:28:51.392 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-14', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:28:51.392 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-14 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:28:51.437 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:28:51.438 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:28:51.520 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:51.522+0000 7fe1763cd900 1 bucket sync caught up with source: 2026-03-20T11:28:51.520 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000002.32.11, , , 00000000002.19.11, 00000000010.49.6, ] 2026-03-20T11:28:51.520 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000002.32.11, , , 00000000002.19.11, 00000000010.49.6, ] 2026-03-20T11:28:51.520 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:51.522+0000 7fe1763cd900 0 bucket checkpoint complete 2026-03-20T11:28:51.530 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-14', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:28:51.530 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-14', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:28:51.530 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-14 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:28:51.570 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:28:51.570 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:28:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:51.649+0000 7ff13c34b900 1 bucket sync caught up with source: 2026-03-20T11:28:51.648 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000003.36.10, , , 00000000010.44.6, 00000000002.40.11, ] 2026-03-20T11:28:51.648 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000003.36.10, , , 00000000010.44.6, 00000000002.40.11, ] 2026-03-20T11:28:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:51.649+0000 7ff13c34b900 0 bucket checkpoint complete 2026-03-20T11:28:51.658 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-14 zones={a2, a1} 2026-03-20T11:28:51.667 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:28:51.667 DEBUG:tasks.rgw_multi.zone_rados:o=dummy, v=CdIw-pGNf7s7MpZAiMuAQnjphxyb7mH 2026-03-20T11:28:51.667 DEBUG:tasks.rgw_multi.zone_rados:o=obj-a1, v=m3xozuaqbNV7vDR1Cj3msfV5U8iKzRr 2026-03-20T11:28:51.667 DEBUG:tasks.rgw_multi.zone_rados:o=obj-a2, v=HNtJL9hmfiuwvoaAbm7MeOpj.lljhDC 2026-03-20T11:28:51.670 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:28:51.670 DEBUG:tasks.rgw_multi.zone_rados:o=dummy, v=CdIw-pGNf7s7MpZAiMuAQnjphxyb7mH 2026-03-20T11:28:51.670 DEBUG:tasks.rgw_multi.zone_rados:o=obj-a1, v=m3xozuaqbNV7vDR1Cj3msfV5U8iKzRr 2026-03-20T11:28:51.670 DEBUG:tasks.rgw_multi.zone_rados:o=obj-a2, v=HNtJL9hmfiuwvoaAbm7MeOpj.lljhDC 2026-03-20T11:28:51.670 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2026-03-20T11:28:51.684 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-a1 2026-03-20T11:28:51.747 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-a2 2026-03-20T11:28:51.807 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-14 zones={a2, a1} 2026-03-20T11:28:51.886 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-13', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:28:51.887 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-13', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:28:51.887 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-13 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:28:51.931 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:28:51.931 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:28:52.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:52.021+0000 7f2763c18900 1 waiting for incremental sync to catch up: 2026-03-20T11:28:52.020 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000003.29.10, , , 00000000002.8.11, 00000000010.34.6, ] 2026-03-20T11:28:52.020 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000003.29.10, , , 00000000002.8.11, 00000000011.37.6, ] 2026-03-20T11:28:57.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:28:57.022+0000 7f2763c18900 1 waiting for incremental sync to catch up: 2026-03-20T11:28:57.021 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000003.29.10, , , 00000000002.8.11, 00000000010.34.6, ] 2026-03-20T11:28:57.021 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000003.29.10, , , 00000000002.8.11, 00000000011.37.6, ] 2026-03-20T11:29:02.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:02.024+0000 7f2763c18900 1 waiting for incremental sync to catch up: 2026-03-20T11:29:02.023 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000003.29.10, , , 00000000002.8.11, 00000000010.34.6, ] 2026-03-20T11:29:02.023 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000003.29.10, , , 00000000002.8.11, 00000000011.37.6, ] 2026-03-20T11:29:07.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:07.025+0000 7f2763c18900 1 waiting for incremental sync to catch up: 2026-03-20T11:29:07.024 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000003.29.10, , , 00000000002.8.11, 00000000010.34.6, ] 2026-03-20T11:29:07.024 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000003.29.10, , , 00000000002.8.11, 00000000011.37.6, ] 2026-03-20T11:29:12.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:12.026+0000 7f2763c18900 1 bucket sync caught up with source: 2026-03-20T11:29:12.025 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000003.29.10, , , 00000000002.8.11, 00000000011.37.6, ] 2026-03-20T11:29:12.025 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000003.29.10, , , 00000000002.8.11, 00000000011.37.6, ] 2026-03-20T11:29:12.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:12.026+0000 7f2763c18900 0 bucket checkpoint complete 2026-03-20T11:29:12.137 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-13', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:29:12.137 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-13', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:29:12.138 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-13 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:29:12.181 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:29:12.181 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:29:12.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:12.284+0000 7ff1fb620900 1 waiting for incremental sync to catch up: 2026-03-20T11:29:12.283 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000002.36.11, , , 00000000011.25.6, 00000000002.17.11, ] 2026-03-20T11:29:12.283 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000002.36.11, , , 00000000011.25.6, 00000000003.20.6, ] 2026-03-20T11:29:17.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:17.285+0000 7ff1fb620900 1 waiting for incremental sync to catch up: 2026-03-20T11:29:17.284 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000002.36.11, , , 00000000011.25.6, 00000000002.17.11, ] 2026-03-20T11:29:17.284 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000002.36.11, , , 00000000011.25.6, 00000000003.20.6, ] 2026-03-20T11:29:22.285 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:22.286+0000 7ff1fb620900 1 waiting for incremental sync to catch up: 2026-03-20T11:29:22.285 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000002.36.11, , , 00000000011.25.6, 00000000002.17.11, ] 2026-03-20T11:29:22.285 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000002.36.11, , , 00000000011.25.6, 00000000003.20.6, ] 2026-03-20T11:29:27.286 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:27.288+0000 7ff1fb620900 1 waiting for incremental sync to catch up: 2026-03-20T11:29:27.286 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000002.36.11, , , 00000000011.25.6, 00000000002.17.11, ] 2026-03-20T11:29:27.286 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000002.36.11, , , 00000000011.25.6, 00000000003.20.6, ] 2026-03-20T11:29:32.287 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:32.289+0000 7ff1fb620900 1 bucket sync caught up with source: 2026-03-20T11:29:32.287 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000002.36.11, , , 00000000011.25.6, 00000000003.20.6, ] 2026-03-20T11:29:32.287 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000002.36.11, , , 00000000011.25.6, 00000000003.20.6, ] 2026-03-20T11:29:32.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:32.289+0000 7ff1fb620900 0 bucket checkpoint complete 2026-03-20T11:29:32.297 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-13 zones={a2, a1} 2026-03-20T11:29:32.303 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:29:32.303 DEBUG:tasks.rgw_multi.zone_rados:o=dummy, v=Xi2I5D1jFJoS21REWf-2D6eyzZVpgFm 2026-03-20T11:29:32.303 DEBUG:tasks.rgw_multi.zone_rados:o=obj-a1, v=13IwQ-odIOdimI1E5hsY2ZSkETyWvuV 2026-03-20T11:29:32.303 DEBUG:tasks.rgw_multi.zone_rados:o=obj-a2, v=jMNSPaa.HpmlL9uZS79AYqRdglaSpdK 2026-03-20T11:29:32.306 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:29:32.306 DEBUG:tasks.rgw_multi.zone_rados:o=dummy, v=Xi2I5D1jFJoS21REWf-2D6eyzZVpgFm 2026-03-20T11:29:32.306 DEBUG:tasks.rgw_multi.zone_rados:o=obj-a1, v=13IwQ-odIOdimI1E5hsY2ZSkETyWvuV 2026-03-20T11:29:32.306 DEBUG:tasks.rgw_multi.zone_rados:o=obj-a2, v=jMNSPaa.HpmlL9uZS79AYqRdglaSpdK 2026-03-20T11:29:32.306 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2026-03-20T11:29:32.318 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-a1 2026-03-20T11:29:32.374 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-a2 2026-03-20T11:29:32.431 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-13 zones={a2, a1} 2026-03-20T11:29:32.431 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-14', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:29:32.431 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-14', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:29:32.431 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-14 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:29:32.474 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:29:32.474 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:29:32.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:32.551+0000 7f0625878900 1 bucket sync caught up with source: 2026-03-20T11:29:32.550 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000002.32.11, , , 00000000003.22.6, 00000000011.52.6, ] 2026-03-20T11:29:32.550 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000002.32.11, , , 00000000003.22.6, 00000000011.52.6, ] 2026-03-20T11:29:32.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:32.551+0000 7f0625878900 0 bucket checkpoint complete 2026-03-20T11:29:32.559 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-14', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:29:32.559 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-14', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:29:32.559 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-14 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:29:32.603 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:29:32.604 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:29:32.676 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:32.678+0000 7fc6ab611900 1 bucket sync caught up with source: 2026-03-20T11:29:32.676 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000003.36.10, , , 00000000011.47.6, 00000000003.43.6, ] 2026-03-20T11:29:32.676 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000003.36.10, , , 00000000011.47.6, 00000000003.43.6, ] 2026-03-20T11:29:32.676 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:32.678+0000 7fc6ab611900 0 bucket checkpoint complete 2026-03-20T11:29:32.687 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-14 zones={a2, a1} 2026-03-20T11:29:32.693 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:29:32.693 DEBUG:tasks.rgw_multi.zone_rados:o=dummy, v=CdIw-pGNf7s7MpZAiMuAQnjphxyb7mH 2026-03-20T11:29:32.693 DEBUG:tasks.rgw_multi.zone_rados:o=obj-a1, v=m3xozuaqbNV7vDR1Cj3msfV5U8iKzRr 2026-03-20T11:29:32.693 DEBUG:tasks.rgw_multi.zone_rados:o=obj-a2, v=HNtJL9hmfiuwvoaAbm7MeOpj.lljhDC 2026-03-20T11:29:32.696 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:29:32.696 DEBUG:tasks.rgw_multi.zone_rados:o=dummy, v=CdIw-pGNf7s7MpZAiMuAQnjphxyb7mH 2026-03-20T11:29:32.696 DEBUG:tasks.rgw_multi.zone_rados:o=obj-a1, v=m3xozuaqbNV7vDR1Cj3msfV5U8iKzRr 2026-03-20T11:29:32.696 DEBUG:tasks.rgw_multi.zone_rados:o=obj-a2, v=HNtJL9hmfiuwvoaAbm7MeOpj.lljhDC 2026-03-20T11:29:32.696 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2026-03-20T11:29:32.708 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-a1 2026-03-20T11:29:32.764 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-a2 2026-03-20T11:29:32.819 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-14 zones={a2, a1} 2026-03-20T11:29:32.821 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_versioned_object_incremental_sync ... ok 2026-03-20T11:29:33.014 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:29:33.014 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:29:33.014 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:29:33.051 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:29:33.051 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:29:33.123 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:29:33.136 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774006173.014278_74.1","last_update":"2026-03-20T11:29:33.014278Z"},{"marker":"1_1774006026.822813_82.1","last_update":"2026-03-20T11:27:06.822813Z"}] 2026-03-20T11:29:33.136 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:29:33.136 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:29:33.136 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:29:33.136 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:29:33.213 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:29:33.213 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:29:33.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:33.285+0000 7fd29cf41900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:29:33.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:33.286+0000 7fd29cf41900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:29:33.294 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006173.014278_74.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:29:33.014278Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006026.822813_82.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.822813Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:29:33Z"} 2026-03-20T11:29:33.294 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:29:33.302 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-15', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:29:33.302 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-15', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:29:33.302 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-15 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:29:33.375 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:29:33.375 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:29:33.449 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:33.450+0000 7f056a615900 1 waiting to reach incremental sync.. 2026-03-20T11:29:38.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:38.452+0000 7f056a615900 1 bucket sync caught up with source: 2026-03-20T11:29:38.451 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.9.6, , ] 2026-03-20T11:29:38.451 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000001.9.6, , ] 2026-03-20T11:29:38.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:38.452+0000 7f056a615900 0 bucket checkpoint complete 2026-03-20T11:29:38.461 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-15', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:29:38.461 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-15', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:29:38.461 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-15 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:29:38.552 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:29:38.553 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:29:38.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:38.632+0000 7f39b0b28900 1 waiting to reach incremental sync.. 2026-03-20T11:29:43.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:43.634+0000 7f39b0b28900 1 bucket sync caught up with source: 2026-03-20T11:29:43.632 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.3.6, , ] 2026-03-20T11:29:43.633 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000001.3.6, , ] 2026-03-20T11:29:43.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:43.634+0000 7f39b0b28900 0 bucket checkpoint complete 2026-03-20T11:29:43.641 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-15 zones={a2, a1} 2026-03-20T11:29:43.652 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:29:43.652 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=null 2026-03-20T11:29:43.656 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:29:43.656 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=null 2026-03-20T11:29:43.656 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:29:43.672 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-15 zones={a2, a1} 2026-03-20T11:29:43.677 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:29:43.677 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:29:43.677 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:29:43.716 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:29:43.716 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:29:43.794 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:29:43.804 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774006183.678829_77.1","last_update":"2026-03-20T11:29:43.678829Z"},{"marker":"1_1774006026.822813_82.1","last_update":"2026-03-20T11:27:06.822813Z"}] 2026-03-20T11:29:43.804 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:29:43.804 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:29:43.804 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:29:43.804 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:29:43.884 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:29:43.884 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:29:43.959 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:43.961+0000 7ffb5b52a900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:29:43.960 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:43.962+0000 7ffb5b52a900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:29:43.970 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006183.678829_77.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:29:43.678829Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006026.822813_82.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.822813Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:29:43Z"} 2026-03-20T11:29:43.970 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:29:43.982 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-15', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:29:43.982 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-15', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:29:43.982 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-15 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:29:44.054 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:29:44.054 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:29:44.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:44.214+0000 7f51f8f2f900 1 waiting for incremental sync to catch up: 2026-03-20T11:29:44.213 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.9.6, , ] 2026-03-20T11:29:44.213 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000003.12.13, , ] 2026-03-20T11:29:49.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:49.215+0000 7f51f8f2f900 1 waiting for incremental sync to catch up: 2026-03-20T11:29:49.214 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.9.6, , ] 2026-03-20T11:29:49.214 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000003.12.13, , ] 2026-03-20T11:29:54.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:54.216+0000 7f51f8f2f900 1 bucket sync caught up with source: 2026-03-20T11:29:54.215 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000003.12.13, , ] 2026-03-20T11:29:54.215 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000003.12.13, , ] 2026-03-20T11:29:54.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:54.216+0000 7f51f8f2f900 0 bucket checkpoint complete 2026-03-20T11:29:54.225 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-15', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:29:54.225 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-15', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:29:54.225 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-15 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:29:54.269 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:29:54.269 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:29:54.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:54.345+0000 7f8536c20900 1 waiting for incremental sync to catch up: 2026-03-20T11:29:54.343 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.3.6, , ] 2026-03-20T11:29:54.343 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000003.7.13, , ] 2026-03-20T11:29:59.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:29:59.346+0000 7f8536c20900 1 waiting for incremental sync to catch up: 2026-03-20T11:29:59.344 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.3.6, , ] 2026-03-20T11:29:59.344 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000003.7.13, , ] 2026-03-20T11:30:04.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:30:04.347+0000 7f8536c20900 1 waiting for incremental sync to catch up: 2026-03-20T11:30:04.346 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.3.6, , ] 2026-03-20T11:30:04.346 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000003.7.13, , ] 2026-03-20T11:30:09.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:30:09.348+0000 7f8536c20900 1 waiting for incremental sync to catch up: 2026-03-20T11:30:09.347 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.3.6, , ] 2026-03-20T11:30:09.347 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000003.7.13, , ] 2026-03-20T11:30:14.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:30:14.349+0000 7f8536c20900 1 bucket sync caught up with source: 2026-03-20T11:30:14.348 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000003.7.13, , ] 2026-03-20T11:30:14.348 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000003.7.13, , ] 2026-03-20T11:30:14.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:30:14.349+0000 7f8536c20900 0 bucket checkpoint complete 2026-03-20T11:30:14.356 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-15 zones={a2, a1} 2026-03-20T11:30:14.363 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:30:14.363 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=ipPRBB77jCfe.h7-VWmNOWCftX1Ym.H 2026-03-20T11:30:14.363 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=null 2026-03-20T11:30:14.366 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:30:14.366 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=ipPRBB77jCfe.h7-VWmNOWCftX1Ym.H 2026-03-20T11:30:14.366 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=null 2026-03-20T11:30:14.366 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:30:14.378 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:30:14.389 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-15 zones={a2, a1} 2026-03-20T11:30:14.409 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-15', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:30:14.409 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-15', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:30:14.409 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-15 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:30:14.449 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:30:14.449 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:30:14.519 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:30:14.520+0000 7fb1b2a20900 1 waiting for incremental sync to catch up: 2026-03-20T11:30:14.519 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000003.12.13, , ] 2026-03-20T11:30:14.519 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000007.20.6, , ] 2026-03-20T11:30:19.520 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:30:19.521+0000 7fb1b2a20900 1 waiting for incremental sync to catch up: 2026-03-20T11:30:19.520 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000003.12.13, , ] 2026-03-20T11:30:19.520 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000007.20.6, , ] 2026-03-20T11:30:24.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:30:24.523+0000 7fb1b2a20900 1 waiting for incremental sync to catch up: 2026-03-20T11:30:24.521 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000003.12.13, , ] 2026-03-20T11:30:24.521 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000007.20.6, , ] 2026-03-20T11:30:29.522 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:30:29.524+0000 7fb1b2a20900 1 waiting for incremental sync to catch up: 2026-03-20T11:30:29.522 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000003.12.13, , ] 2026-03-20T11:30:29.522 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000007.20.6, , ] 2026-03-20T11:30:34.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:30:34.524+0000 7fb1b2a20900 1 waiting for incremental sync to catch up: 2026-03-20T11:30:34.523 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000003.12.13, , ] 2026-03-20T11:30:34.523 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000007.20.6, , ] 2026-03-20T11:30:39.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:30:39.526+0000 7fb1b2a20900 1 bucket sync caught up with source: 2026-03-20T11:30:39.524 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000007.20.6, , ] 2026-03-20T11:30:39.524 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000007.20.6, , ] 2026-03-20T11:30:39.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:30:39.526+0000 7fb1b2a20900 0 bucket checkpoint complete 2026-03-20T11:30:39.534 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-15', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:30:39.534 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-15', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:30:39.534 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-15 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:30:39.618 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:30:39.618 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:30:39.700 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:30:39.701+0000 7f9cf8922900 1 waiting for incremental sync to catch up: 2026-03-20T11:30:39.700 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000003.7.13, , ] 2026-03-20T11:30:39.700 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000007.15.6, , ] 2026-03-20T11:30:44.701 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:30:44.703+0000 7f9cf8922900 1 waiting for incremental sync to catch up: 2026-03-20T11:30:44.701 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000003.7.13, , ] 2026-03-20T11:30:44.701 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000007.15.6, , ] 2026-03-20T11:30:49.702 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:30:49.705+0000 7f9cf8922900 1 waiting for incremental sync to catch up: 2026-03-20T11:30:49.702 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000003.7.13, , ] 2026-03-20T11:30:49.702 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000007.15.6, , ] 2026-03-20T11:30:54.703 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:30:54.706+0000 7f9cf8922900 1 waiting for incremental sync to catch up: 2026-03-20T11:30:54.703 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000003.7.13, , ] 2026-03-20T11:30:54.703 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000007.15.6, , ] 2026-03-20T11:30:59.704 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:30:59.707+0000 7f9cf8922900 1 bucket sync caught up with source: 2026-03-20T11:30:59.704 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000007.15.6, , ] 2026-03-20T11:30:59.704 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000007.15.6, , ] 2026-03-20T11:30:59.704 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:30:59.707+0000 7f9cf8922900 0 bucket checkpoint complete 2026-03-20T11:30:59.713 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-15 zones={a2, a1} 2026-03-20T11:30:59.719 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:30:59.722 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:30:59.722 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-15 zones={a2, a1} 2026-03-20T11:30:59.725 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_null_version_id_delete ... ok 2026-03-20T11:30:59.790 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:30:59.790 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:30:59.791 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:30:59.827 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:30:59.827 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:30:59.897 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:30:59.908 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774006259.792589_80.1","last_update":"2026-03-20T11:30:59.792589Z"},{"marker":"1_1774006026.822813_82.1","last_update":"2026-03-20T11:27:06.822813Z"}] 2026-03-20T11:30:59.909 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:30:59.909 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:30:59.909 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:30:59.909 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:30:59.990 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:30:59.990 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:31:00.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:00.074+0000 7eff3d328900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:31:00.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:00.075+0000 7eff3d328900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:31:00.085 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006259.792589_80.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:30:59.792589Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006026.822813_82.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.822813Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:31:00Z"} 2026-03-20T11:31:00.085 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:31:00.101 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-16', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:31:00.101 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-16', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:31:00.101 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-16 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:31:00.167 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:31:00.168 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:31:00.247 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:00.249+0000 7f692092e900 1 waiting to reach incremental sync.. 2026-03-20T11:31:05.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:05.251+0000 7f692092e900 1 bucket sync caught up with source: 2026-03-20T11:31:05.248 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000002.15.11, , , , , ] 2026-03-20T11:31:05.248 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000002.15.11, , , , , ] 2026-03-20T11:31:05.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:05.251+0000 7f692092e900 0 bucket checkpoint complete 2026-03-20T11:31:05.258 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-16', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:31:05.258 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-16', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:31:05.258 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-16 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:31:05.339 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:31:05.339 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:31:05.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:05.414+0000 7f2d90139900 1 bucket sync caught up with source: 2026-03-20T11:31:05.412 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000002.22.11, , , , , ] 2026-03-20T11:31:05.412 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000002.22.11, , , , , ] 2026-03-20T11:31:05.412 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:05.414+0000 7f2d90139900 0 bucket checkpoint complete 2026-03-20T11:31:05.421 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-16 zones={a2, a1} 2026-03-20T11:31:05.429 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:31:05.429 DEBUG:tasks.rgw_multi.zone_rados:o=dummy, v=h4ZMoxAf4z1H3GaRY7HlJRAhkA56mOP 2026-03-20T11:31:05.432 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:31:05.432 DEBUG:tasks.rgw_multi.zone_rados:o=dummy, v=h4ZMoxAf4z1H3GaRY7HlJRAhkA56mOP 2026-03-20T11:31:05.432 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2026-03-20T11:31:05.486 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-16 zones={a2, a1} 2026-03-20T11:31:05.737 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-16', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:31:05.737 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-16', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:31:05.737 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-16 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:31:05.780 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:31:05.780 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:31:05.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:05.865+0000 7f6ca5f28900 1 waiting for incremental sync to catch up: 2026-03-20T11:31:05.863 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000002.15.11, , , , , ] 2026-03-20T11:31:05.863 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000003.17.10, , , 00000000020.62.12, , ] 2026-03-20T11:31:10.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:10.866+0000 7f6ca5f28900 1 waiting for incremental sync to catch up: 2026-03-20T11:31:10.864 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000002.15.11, , , 00000000020.62.12, , ] 2026-03-20T11:31:10.864 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000003.17.10, , , 00000000020.62.12, , ] 2026-03-20T11:31:15.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:15.867+0000 7f6ca5f28900 1 waiting for incremental sync to catch up: 2026-03-20T11:31:15.865 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000002.15.11, , , 00000000020.62.12, , ] 2026-03-20T11:31:15.865 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000003.17.10, , , 00000000020.62.12, , ] 2026-03-20T11:31:20.866 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:20.868+0000 7f6ca5f28900 1 waiting for incremental sync to catch up: 2026-03-20T11:31:20.866 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000002.15.11, , , 00000000020.62.12, , ] 2026-03-20T11:31:20.866 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000003.17.10, , , 00000000020.62.12, , ] 2026-03-20T11:31:25.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:25.869+0000 7f6ca5f28900 1 bucket sync caught up with source: 2026-03-20T11:31:25.867 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000003.17.10, , , 00000000031.93.9, , ] 2026-03-20T11:31:25.867 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000003.17.10, , , 00000000020.62.12, , ] 2026-03-20T11:31:25.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:25.869+0000 7f6ca5f28900 0 bucket checkpoint complete 2026-03-20T11:31:25.922 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-16', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:31:25.922 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-16', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:31:25.922 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-16 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:31:25.964 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:31:25.964 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:31:26.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:26.055+0000 7fdf48725900 1 bucket sync caught up with source: 2026-03-20T11:31:26.053 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000002.22.11, , , 00000000031.119.12, , ] 2026-03-20T11:31:26.053 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000002.22.11, , , 00000000031.119.12, , ] 2026-03-20T11:31:26.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:26.055+0000 7fdf48725900 0 bucket checkpoint complete 2026-03-20T11:31:26.064 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-16 zones={a2, a1} 2026-03-20T11:31:26.073 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:31:26.073 DEBUG:tasks.rgw_multi.zone_rados:o=dummy, v=h4ZMoxAf4z1H3GaRY7HlJRAhkA56mOP 2026-03-20T11:31:26.073 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=MxhYVhfEEQCIFNb1hCJ-Xuz6QOSrLpi 2026-03-20T11:31:26.073 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=dKsgDfZI4pS66fvIQByJRM8AigzbCux 2026-03-20T11:31:26.073 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=Hm9JkF0HjaVTmMKvSbNhSz0JLGTS83p 2026-03-20T11:31:26.073 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=OPsbsXcMu9zkkTeSj1rOHPlzZmJYfOy 2026-03-20T11:31:26.073 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=JqvtzOGAibGwVUXIs35DdnHcgXV27Fk 2026-03-20T11:31:26.073 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=depkOvHhOi.8FeIJrKebUdQhijT03Pu 2026-03-20T11:31:26.073 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=0SFGXxGg2ZbchPWk7Y4F6nL42rLDSFv 2026-03-20T11:31:26.073 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=N-X1Pyiu-otTIEKB2z.Cw98WDVkSuyb 2026-03-20T11:31:26.073 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=A6gfDnawsGmyOduhG9vC0eBJ7T0OQBi 2026-03-20T11:31:26.073 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=D0sipD1iGlo-XB1EM33wksyO-NaJA23 2026-03-20T11:31:26.073 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=hiaAvs6tvvztj9nPwO1bJ0Ks6XZ2BLE 2026-03-20T11:31:26.073 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=t6bamXCGebF9jtZDj1cYUnUptVU7iy- 2026-03-20T11:31:26.073 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=a-EqgaUDvlntfUjQ5AQIJNLWvKCsC5- 2026-03-20T11:31:26.073 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=ov2OnfcMRXILaC4Ei.YGx8-AdU7kRmp 2026-03-20T11:31:26.073 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=Lkl.vT9VHYJKhOjcAs1mr5VtrVBCaB. 2026-03-20T11:31:26.073 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=svmgJ2KyMIH60juoceJSLTYyBH8y6AM 2026-03-20T11:31:26.073 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=9tB--1YcEEh7hR.qYtpAc3NIcPEcdEe 2026-03-20T11:31:26.073 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=dUeX6jv9T3oVQQXbIIm7Z64PHHX1Olo 2026-03-20T11:31:26.073 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=CIHJDRGwlcHtHRLBVsPaJmVCevl3mqN 2026-03-20T11:31:26.073 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=HeiIi4nx4TbpMNxBBphlRGRME9f0jnV 2026-03-20T11:31:26.077 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:31:26.077 DEBUG:tasks.rgw_multi.zone_rados:o=dummy, v=h4ZMoxAf4z1H3GaRY7HlJRAhkA56mOP 2026-03-20T11:31:26.077 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=MxhYVhfEEQCIFNb1hCJ-Xuz6QOSrLpi 2026-03-20T11:31:26.077 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=dKsgDfZI4pS66fvIQByJRM8AigzbCux 2026-03-20T11:31:26.077 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=Hm9JkF0HjaVTmMKvSbNhSz0JLGTS83p 2026-03-20T11:31:26.077 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=OPsbsXcMu9zkkTeSj1rOHPlzZmJYfOy 2026-03-20T11:31:26.077 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=JqvtzOGAibGwVUXIs35DdnHcgXV27Fk 2026-03-20T11:31:26.077 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=depkOvHhOi.8FeIJrKebUdQhijT03Pu 2026-03-20T11:31:26.077 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=0SFGXxGg2ZbchPWk7Y4F6nL42rLDSFv 2026-03-20T11:31:26.077 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=N-X1Pyiu-otTIEKB2z.Cw98WDVkSuyb 2026-03-20T11:31:26.077 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=A6gfDnawsGmyOduhG9vC0eBJ7T0OQBi 2026-03-20T11:31:26.077 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=D0sipD1iGlo-XB1EM33wksyO-NaJA23 2026-03-20T11:31:26.077 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=hiaAvs6tvvztj9nPwO1bJ0Ks6XZ2BLE 2026-03-20T11:31:26.077 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=t6bamXCGebF9jtZDj1cYUnUptVU7iy- 2026-03-20T11:31:26.077 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=a-EqgaUDvlntfUjQ5AQIJNLWvKCsC5- 2026-03-20T11:31:26.077 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=ov2OnfcMRXILaC4Ei.YGx8-AdU7kRmp 2026-03-20T11:31:26.077 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=Lkl.vT9VHYJKhOjcAs1mr5VtrVBCaB. 2026-03-20T11:31:26.077 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=svmgJ2KyMIH60juoceJSLTYyBH8y6AM 2026-03-20T11:31:26.077 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=9tB--1YcEEh7hR.qYtpAc3NIcPEcdEe 2026-03-20T11:31:26.077 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=dUeX6jv9T3oVQQXbIIm7Z64PHHX1Olo 2026-03-20T11:31:26.077 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=CIHJDRGwlcHtHRLBVsPaJmVCevl3mqN 2026-03-20T11:31:26.077 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=HeiIi4nx4TbpMNxBBphlRGRME9f0jnV 2026-03-20T11:31:26.077 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2026-03-20T11:31:26.091 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:31:26.155 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:31:26.218 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:31:26.275 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:31:26.337 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:31:26.397 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:31:26.457 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:31:26.516 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:31:26.578 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:31:26.640 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:31:26.702 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:31:26.763 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:31:26.827 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:31:26.889 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:31:26.945 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:31:27.005 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:31:27.063 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:31:27.117 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:31:27.177 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:31:27.233 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:31:27.291 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-16 zones={a2, a1} 2026-03-20T11:31:27.291 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:31:27.291 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:31:27.291 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:31:27.331 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:31:27.331 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:31:27.419 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"00000000000000000000:00000000000000003357","last_update":"2026-03-20T11:28:44.807706Z"},{"marker":"00000000000000000000:00000000000000006733","last_update":"2026-03-20T11:31:18.834043Z"},{"marker":"00000000000000000000:00000000000000006319","last_update":"2026-03-20T11:31:18.835901Z"},{"marker":"00000000000000000000:00000000000000003914","last_update":"2026-03-20T11:28:44.813049Z"}] 2026-03-20T11:31:27.419 INFO:rgw_multi.tests:starting data checkpoint for target_zone=a2 source_zone=a1 2026-03-20T11:31:27.419 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:31:27.419 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:31:27.419 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --source-zone a1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:31:27.500 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:31:27.500 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:31:27.581 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"instance_id":15702199979187085538},"markers":[{"key":0,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000003357","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:28:44.807706Z"}},{"key":1,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000006733","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:31:18.834043Z"}},{"key":2,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000006319","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:31:18.835901Z"}},{"key":3,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000003914","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:28:44.813049Z"}}]},"full_sync":{"total":0,"complete":0},"current_time":"2026-03-20T11:31:27Z"} 2026-03-20T11:31:27.581 INFO:rgw_multi.tests:finished data checkpoint for target_zone=a2 source_zone=a1 2026-03-20T11:31:27.581 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:31:27.581 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:31:27.581 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:31:27.659 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:31:27.670 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:31:27.993 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"00000000000000000000:00000000000000003639","last_update":"2026-03-20T11:28:20.439213Z"},{"marker":"00000000000000000000:00000000000000007156","last_update":"2026-03-20T11:31:16.628262Z"},{"marker":"00000000000000000000:00000000000000006742","last_update":"2026-03-20T11:31:16.629414Z"},{"marker":"00000000000000000000:00000000000000004338","last_update":"2026-03-20T11:28:20.445084Z"}] 2026-03-20T11:31:27.993 INFO:rgw_multi.tests:starting data checkpoint for target_zone=a1 source_zone=a2 2026-03-20T11:31:27.993 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:31:27.993 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:31:27.993 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --source-zone a2 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:31:28.071 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:31:28.071 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:31:28.158 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"instance_id":5866003269306126155},"markers":[{"key":0,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000003639","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:28:20.439213Z"}},{"key":1,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000007156","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:31:16.628262Z"}},{"key":2,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000006742","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:31:16.629414Z"}},{"key":3,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000004338","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:28:20.445084Z"}}]},"full_sync":{"total":0,"complete":0},"current_time":"2026-03-20T11:31:28Z"} 2026-03-20T11:31:28.158 INFO:rgw_multi.tests:finished data checkpoint for target_zone=a1 source_zone=a2 2026-03-20T11:31:28.161 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_concurrent_versioned_object_incremental_sync ... ok 2026-03-20T11:31:28.301 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:31:28.301 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:31:28.301 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:31:28.337 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:31:28.337 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:31:28.416 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:31:28.426 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774006288.301874_82.1","last_update":"2026-03-20T11:31:28.301874Z"},{"marker":"1_1774006026.822813_82.1","last_update":"2026-03-20T11:27:06.822813Z"}] 2026-03-20T11:31:28.426 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:31:28.427 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:31:28.427 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:31:28.427 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:31:28.505 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:31:28.505 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:31:28.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:28.592+0000 7f8c0d12c900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:31:28.591 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:28.593+0000 7f8c0d12c900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:31:28.600 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006288.301874_82.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:31:28.301874Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006026.822813_82.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.822813Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:31:28Z"} 2026-03-20T11:31:28.601 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:31:28.609 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-17', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:31:28.609 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-17', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:31:28.609 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-17 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:31:28.690 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:31:28.690 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:31:28.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:28.771+0000 7f9c6b162900 1 waiting to reach incremental sync.. 2026-03-20T11:31:33.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:33.772+0000 7f9c6b162900 1 bucket sync caught up with source: 2026-03-20T11:31:33.770 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.39.6, , ] 2026-03-20T11:31:33.770 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000001.39.6, , ] 2026-03-20T11:31:33.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:33.772+0000 7f9c6b162900 0 bucket checkpoint complete 2026-03-20T11:31:33.779 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-17', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:31:33.779 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-17', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:31:33.780 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-17 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:31:33.820 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:31:33.820 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:31:33.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:33.903+0000 7f2fc69cc900 1 waiting to reach incremental sync.. 2026-03-20T11:31:38.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:38.904+0000 7f2fc69cc900 1 bucket sync caught up with source: 2026-03-20T11:31:38.902 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.40.6, , ] 2026-03-20T11:31:38.902 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000001.40.6, , ] 2026-03-20T11:31:38.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:38.904+0000 7f2fc69cc900 0 bucket checkpoint complete 2026-03-20T11:31:38.914 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-17 zones={a2, a1} 2026-03-20T11:31:38.926 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:31:38.927 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=null 2026-03-20T11:31:38.930 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:31:38.931 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=null 2026-03-20T11:31:38.931 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:31:38.945 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-17 zones={a2, a1} 2026-03-20T11:31:38.950 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:31:38.951 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:31:38.951 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:31:38.994 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:31:38.994 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:31:39.066 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:31:39.075 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774006298.952438_83.1","last_update":"2026-03-20T11:31:38.952438Z"},{"marker":"1_1774006026.822813_82.1","last_update":"2026-03-20T11:27:06.822813Z"}] 2026-03-20T11:31:39.075 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:31:39.075 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:31:39.075 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:31:39.076 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:31:39.155 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:31:39.155 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:31:39.302 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:39.305+0000 7f3370b2c900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:31:39.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:39.306+0000 7f3370b2c900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:31:39.313 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006298.952438_83.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:31:38.952438Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006026.822813_82.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.822813Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:31:39Z"} 2026-03-20T11:31:39.314 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:31:39.325 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-17', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:31:39.325 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-17', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:31:39.325 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-17 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:31:39.394 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:31:39.394 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:31:39.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:39.472+0000 7f2f63b2a900 1 waiting for incremental sync to catch up: 2026-03-20T11:31:39.469 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.39.6, , ] 2026-03-20T11:31:39.469 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000003.42.13, , ] 2026-03-20T11:31:44.471 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:44.473+0000 7f2f63b2a900 1 bucket sync caught up with source: 2026-03-20T11:31:44.471 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000003.42.13, , ] 2026-03-20T11:31:44.471 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000003.42.13, , ] 2026-03-20T11:31:44.471 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:44.473+0000 7f2f63b2a900 0 bucket checkpoint complete 2026-03-20T11:31:44.480 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-17', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:31:44.480 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-17', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:31:44.480 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-17 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:31:44.563 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:31:44.563 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:31:44.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:44.648+0000 7f42a0124900 1 waiting for incremental sync to catch up: 2026-03-20T11:31:44.645 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.40.6, , ] 2026-03-20T11:31:44.645 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000003.45.13, , ] 2026-03-20T11:31:49.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:49.649+0000 7f42a0124900 1 waiting for incremental sync to catch up: 2026-03-20T11:31:49.646 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.40.6, , ] 2026-03-20T11:31:49.646 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000003.45.13, , ] 2026-03-20T11:31:54.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:54.650+0000 7f42a0124900 1 waiting for incremental sync to catch up: 2026-03-20T11:31:54.647 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.40.6, , ] 2026-03-20T11:31:54.647 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000003.45.13, , ] 2026-03-20T11:31:59.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:31:59.651+0000 7f42a0124900 1 waiting for incremental sync to catch up: 2026-03-20T11:31:59.649 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.40.6, , ] 2026-03-20T11:31:59.649 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000003.45.13, , ] 2026-03-20T11:32:04.165 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:32:04.168+0000 7f11de8ad640 -1 WARNING: The OSD cluster is overloaded and struggling to complete ops. You need more capacity to serve this level of demand. 2026-03-20T11:32:04.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:32:04.652+0000 7f42a0124900 1 bucket sync caught up with source: 2026-03-20T11:32:04.649 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000003.45.13, , ] 2026-03-20T11:32:04.650 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000003.45.13, , ] 2026-03-20T11:32:04.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:32:04.652+0000 7f42a0124900 0 bucket checkpoint complete 2026-03-20T11:32:04.659 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-17 zones={a2, a1} 2026-03-20T11:32:04.665 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:32:04.665 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=GTkrXMyeUQRlI4h0kFCMenZNwxqB9G5 2026-03-20T11:32:04.665 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=null 2026-03-20T11:32:04.667 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:32:04.667 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=GTkrXMyeUQRlI4h0kFCMenZNwxqB9G5 2026-03-20T11:32:04.667 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=null 2026-03-20T11:32:04.667 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:32:04.679 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:32:04.691 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-17 zones={a2, a1} 2026-03-20T11:32:04.695 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:32:04.696 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:32:04.696 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:32:04.736 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:32:04.736 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:32:04.807 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:32:04.817 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774006324.697809_84.1","last_update":"2026-03-20T11:32:04.697809Z"},{"marker":"1_1774006026.822813_82.1","last_update":"2026-03-20T11:27:06.822813Z"}] 2026-03-20T11:32:04.817 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:32:04.817 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:32:04.817 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:32:04.817 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:32:04.899 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:32:04.899 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:32:04.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:32:04.972+0000 7fb5f2732900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:32:04.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:32:04.973+0000 7fb5f2732900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:32:04.980 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006324.697809_84.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:32:04.697809Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006026.822813_82.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.822813Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:32:04Z"} 2026-03-20T11:32:04.980 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:32:04.991 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-17', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:32:04.992 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-17', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:32:04.992 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-17 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:32:05.060 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:32:05.060 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:32:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:32:05.136+0000 7f3da5a10900 1 waiting for incremental sync to catch up: 2026-03-20T11:32:05.133 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000003.42.13, , ] 2026-03-20T11:32:05.133 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000005.46.18, , ] 2026-03-20T11:32:10.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:32:10.137+0000 7f3da5a10900 1 waiting for incremental sync to catch up: 2026-03-20T11:32:10.135 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000003.42.13, , ] 2026-03-20T11:32:10.135 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000005.46.18, , ] 2026-03-20T11:32:15.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:32:15.138+0000 7f3da5a10900 1 waiting for incremental sync to catch up: 2026-03-20T11:32:15.136 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000003.42.13, , ] 2026-03-20T11:32:15.136 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000005.46.18, , ] 2026-03-20T11:32:20.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:32:20.139+0000 7f3da5a10900 1 waiting for incremental sync to catch up: 2026-03-20T11:32:20.137 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000003.42.13, , ] 2026-03-20T11:32:20.137 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000005.46.18, , ] 2026-03-20T11:32:25.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:32:25.140+0000 7f3da5a10900 1 waiting for incremental sync to catch up: 2026-03-20T11:32:25.138 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000003.42.13, , ] 2026-03-20T11:32:25.138 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000005.46.18, , ] 2026-03-20T11:32:30.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:32:30.141+0000 7f3da5a10900 1 bucket sync caught up with source: 2026-03-20T11:32:30.139 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000005.46.18, , ] 2026-03-20T11:32:30.139 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000005.46.18, , ] 2026-03-20T11:32:30.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:32:30.141+0000 7f3da5a10900 0 bucket checkpoint complete 2026-03-20T11:32:30.148 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-17', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:32:30.148 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-17', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:32:30.148 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-17 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:32:30.230 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:32:30.230 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:32:30.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:32:30.307+0000 7fc4f513b900 1 waiting for incremental sync to catch up: 2026-03-20T11:32:30.305 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000003.45.13, , ] 2026-03-20T11:32:30.305 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000005.51.18, , ] 2026-03-20T11:32:35.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:32:35.308+0000 7fc4f513b900 1 waiting for incremental sync to catch up: 2026-03-20T11:32:35.306 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000003.45.13, , ] 2026-03-20T11:32:35.306 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000005.51.18, , ] 2026-03-20T11:32:40.307 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:32:40.309+0000 7fc4f513b900 1 waiting for incremental sync to catch up: 2026-03-20T11:32:40.307 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000003.45.13, , ] 2026-03-20T11:32:40.307 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000005.51.18, , ] 2026-03-20T11:32:45.308 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:32:45.310+0000 7fc4f513b900 1 waiting for incremental sync to catch up: 2026-03-20T11:32:45.308 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000003.45.13, , ] 2026-03-20T11:32:45.308 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000005.51.18, , ] 2026-03-20T11:32:50.309 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:32:50.312+0000 7fc4f513b900 1 bucket sync caught up with source: 2026-03-20T11:32:50.309 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000005.51.18, , ] 2026-03-20T11:32:50.309 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000005.51.18, , ] 2026-03-20T11:32:50.309 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:32:50.312+0000 7fc4f513b900 0 bucket checkpoint complete 2026-03-20T11:32:50.410 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-17 zones={a2, a1} 2026-03-20T11:32:50.417 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:32:50.417 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=null 2026-03-20T11:32:50.417 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=GTkrXMyeUQRlI4h0kFCMenZNwxqB9G5 2026-03-20T11:32:50.420 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:32:50.420 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=null 2026-03-20T11:32:50.420 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=GTkrXMyeUQRlI4h0kFCMenZNwxqB9G5 2026-03-20T11:32:50.420 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:32:50.434 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:32:50.447 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-17 zones={a2, a1} 2026-03-20T11:32:50.450 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_version_suspended_incremental_sync ... ok 2026-03-20T11:32:50.516 INFO:rgw_multi.tests:create bucket zone=a1 name=dipepu-18 2026-03-20T11:32:50.760 INFO:rgw_multi.tests:create bucket zone=a2 name=dipepu-19 2026-03-20T11:32:50.844 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:32:50.844 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:32:50.844 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:32:50.884 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:32:50.884 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:32:50.967 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:32:50.977 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774006370.843447_90.1","last_update":"2026-03-20T11:32:50.843447Z"},{"marker":"1_1774006026.822813_82.1","last_update":"2026-03-20T11:27:06.822813Z"}] 2026-03-20T11:32:50.977 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:32:50.977 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:32:50.978 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:32:50.978 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:32:51.058 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:32:51.058 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:32:51.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:32:51.139+0000 7f91e692c900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:32:51.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:32:51.140+0000 7f91e692c900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:32:51.148 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006370.843447_90.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:32:50.843447Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006026.822813_82.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.822813Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:32:51Z"} 2026-03-20T11:32:51.148 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:32:51.240 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-18', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:32:51.240 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-18', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:32:51.240 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-18 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:32:51.284 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:32:51.284 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:32:51.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:32:51.391+0000 7fa903341900 1 waiting to reach incremental sync.. 2026-03-20T11:32:56.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:32:56.392+0000 7fa903341900 1 bucket sync caught up with source: 2026-03-20T11:32:56.389 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000005.51.12, , ] 2026-03-20T11:32:56.389 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000005.51.12, , ] 2026-03-20T11:32:56.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:32:56.392+0000 7fa903341900 0 bucket checkpoint complete 2026-03-20T11:32:56.561 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-18', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:32:56.562 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-18', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:32:56.562 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-18 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:32:56.603 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:32:56.603 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:32:56.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:32:56.888+0000 7f3a4bb36900 1 bucket sync caught up with source: 2026-03-20T11:32:56.885 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000002.59.6, , ] 2026-03-20T11:32:56.885 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000002.59.6, , ] 2026-03-20T11:32:56.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:32:56.888+0000 7f3a4bb36900 0 bucket checkpoint complete 2026-03-20T11:32:56.895 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-18 zones={a2, a1} 2026-03-20T11:32:56.904 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:32:56.904 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=dgfuv8Rj4FdST5v-Kz.yz-CWhMlPcNJ 2026-03-20T11:32:56.904 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=WkGXytEcgp.SLH-nor61aJMG0dz-sx2 2026-03-20T11:32:56.904 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=5FUQCht1dGMUxFvL92IPQYQsQPf16-A 2026-03-20T11:32:56.904 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=4tw2SpgwPjo3cSzTJROrJcfLcEFLwWk 2026-03-20T11:32:56.907 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:32:56.907 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=dgfuv8Rj4FdST5v-Kz.yz-CWhMlPcNJ 2026-03-20T11:32:56.907 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=WkGXytEcgp.SLH-nor61aJMG0dz-sx2 2026-03-20T11:32:56.907 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=5FUQCht1dGMUxFvL92IPQYQsQPf16-A 2026-03-20T11:32:56.907 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=4tw2SpgwPjo3cSzTJROrJcfLcEFLwWk 2026-03-20T11:32:56.907 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:32:56.921 DEBUG:tasks.rgw_multi.zone_rados:both are delete markers, skipping content comparison 2026-03-20T11:32:56.921 DEBUG:tasks.rgw_multi.zone_rados:both are delete markers, skipping content comparison 2026-03-20T11:32:56.921 DEBUG:tasks.rgw_multi.zone_rados:both are delete markers, skipping content comparison 2026-03-20T11:32:56.921 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-18 zones={a2, a1} 2026-03-20T11:32:56.921 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-19', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:32:56.921 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-19', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:32:56.921 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-19 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:32:56.979 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:32:56.979 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:32:57.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:32:57.067+0000 7fb66174d900 1 waiting to reach incremental sync.. 2026-03-20T11:33:02.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:02.069+0000 7fb66174d900 1 bucket sync caught up with source: 2026-03-20T11:33:02.066 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000003.27.6, , ] 2026-03-20T11:33:02.066 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000003.27.6, , ] 2026-03-20T11:33:02.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:02.069+0000 7fb66174d900 0 bucket checkpoint complete 2026-03-20T11:33:02.077 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-19', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:33:02.077 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-19', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:33:02.077 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-19 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:33:02.117 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:33:02.118 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:33:02.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:02.198+0000 7febb3d86900 1 bucket sync caught up with source: 2026-03-20T11:33:02.196 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000005.35.12, , ] 2026-03-20T11:33:02.196 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000005.35.12, , ] 2026-03-20T11:33:02.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:02.198+0000 7febb3d86900 0 bucket checkpoint complete 2026-03-20T11:33:02.205 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-19 zones={a2, a1} 2026-03-20T11:33:02.213 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:33:02.213 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=B1EFd8Dhk9.xrSpHDjNkWs6p2CPjP77 2026-03-20T11:33:02.213 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=YlbF1B8yeajSWWcwfd.bGRgAmNFpqeZ 2026-03-20T11:33:02.213 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=KnMVhkWrsbyJon-gu8tQcjIzyBLHMbG 2026-03-20T11:33:02.213 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=aCnHgmBFfTQ.Uy6icC4CSX8CZuW3TXn 2026-03-20T11:33:02.215 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:33:02.216 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=B1EFd8Dhk9.xrSpHDjNkWs6p2CPjP77 2026-03-20T11:33:02.216 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=YlbF1B8yeajSWWcwfd.bGRgAmNFpqeZ 2026-03-20T11:33:02.216 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=KnMVhkWrsbyJon-gu8tQcjIzyBLHMbG 2026-03-20T11:33:02.216 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=aCnHgmBFfTQ.Uy6icC4CSX8CZuW3TXn 2026-03-20T11:33:02.216 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:33:02.226 DEBUG:tasks.rgw_multi.zone_rados:both are delete markers, skipping content comparison 2026-03-20T11:33:02.226 DEBUG:tasks.rgw_multi.zone_rados:both are delete markers, skipping content comparison 2026-03-20T11:33:02.226 DEBUG:tasks.rgw_multi.zone_rados:both are delete markers, skipping content comparison 2026-03-20T11:33:02.226 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-19 zones={a2, a1} 2026-03-20T11:33:02.229 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_delete_marker_full_sync ... ok 2026-03-20T11:33:02.275 INFO:rgw_multi.tests:create bucket zone=a1 name=dipepu-20 2026-03-20T11:33:02.288 INFO:rgw_multi.tests:create bucket zone=a2 name=dipepu-21 2026-03-20T11:33:02.333 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:33:02.333 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:33:02.333 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:33:02.370 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:33:02.370 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:33:02.463 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:33:02.475 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774006382.333198_98.1","last_update":"2026-03-20T11:33:02.333198Z"},{"marker":"1_1774006026.822813_82.1","last_update":"2026-03-20T11:27:06.822813Z"}] 2026-03-20T11:33:02.475 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:33:02.475 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:33:02.475 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:33:02.475 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:33:02.554 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:33:02.554 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:33:02.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:02.634+0000 7f5df8f2d900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:33:02.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:02.635+0000 7f5df8f2d900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:33:02.642 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006382.333198_98.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:33:02.333198Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006026.822813_82.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.822813Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:33:02Z"} 2026-03-20T11:33:02.643 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:33:02.709 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-20', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:33:02.709 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-20', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:33:02.709 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-20 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:33:02.753 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:33:02.753 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:33:02.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:02.839+0000 7fc7e1f41900 1 waiting to reach incremental sync.. 2026-03-20T11:33:07.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:07.841+0000 7fc7e1f41900 1 bucket sync caught up with source: 2026-03-20T11:33:07.838 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000005.42.14, , ] 2026-03-20T11:33:07.838 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000005.42.14, , ] 2026-03-20T11:33:07.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:07.841+0000 7fc7e1f41900 0 bucket checkpoint complete 2026-03-20T11:33:07.848 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-20', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:33:07.848 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-20', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:33:07.848 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-20 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:33:07.887 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:33:07.887 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:33:07.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:07.960+0000 7f2f86d3c900 1 waiting to reach incremental sync.. 2026-03-20T11:33:12.959 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:12.962+0000 7f2f86d3c900 1 bucket sync caught up with source: 2026-03-20T11:33:12.959 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.41.14, , ] 2026-03-20T11:33:12.959 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000001.41.14, , ] 2026-03-20T11:33:12.959 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:12.962+0000 7f2f86d3c900 0 bucket checkpoint complete 2026-03-20T11:33:12.969 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-20 zones={a2, a1} 2026-03-20T11:33:12.977 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:33:12.977 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=null 2026-03-20T11:33:12.980 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:33:12.980 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=null 2026-03-20T11:33:12.980 DEBUG:tasks.rgw_multi.zone_rados:both are delete markers, skipping content comparison 2026-03-20T11:33:12.980 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-20 zones={a2, a1} 2026-03-20T11:33:12.980 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-21', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:33:12.980 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-21', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:33:12.980 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-21 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:33:13.051 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:33:13.051 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:33:13.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:13.134+0000 7ff15b420900 1 bucket sync caught up with source: 2026-03-20T11:33:13.132 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.60.14, , ] 2026-03-20T11:33:13.132 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000001.60.14, , ] 2026-03-20T11:33:13.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:13.134+0000 7ff15b420900 0 bucket checkpoint complete 2026-03-20T11:33:13.144 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-21', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:33:13.144 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-21', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:33:13.144 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-21 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:33:13.225 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:33:13.225 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:33:13.302 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:13.305+0000 7fd131d26900 1 bucket sync caught up with source: 2026-03-20T11:33:13.302 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000005.84.14, , ] 2026-03-20T11:33:13.302 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000005.84.14, , ] 2026-03-20T11:33:13.302 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:13.305+0000 7fd131d26900 0 bucket checkpoint complete 2026-03-20T11:33:13.312 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-21 zones={a2, a1} 2026-03-20T11:33:13.319 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:33:13.319 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=null 2026-03-20T11:33:13.321 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:33:13.321 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=null 2026-03-20T11:33:13.321 DEBUG:tasks.rgw_multi.zone_rados:both are delete markers, skipping content comparison 2026-03-20T11:33:13.321 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-21 zones={a2, a1} 2026-03-20T11:33:13.324 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_suspended_delete_marker_full_sync ... ok 2026-03-20T11:33:13.429 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:33:13.429 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:33:13.429 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:33:13.470 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:33:13.470 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:33:13.541 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:33:13.552 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774006393.431744_102.1","last_update":"2026-03-20T11:33:13.431744Z"},{"marker":"1_1774006026.822813_82.1","last_update":"2026-03-20T11:27:06.822813Z"}] 2026-03-20T11:33:13.552 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:33:13.552 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:33:13.552 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:33:13.552 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:33:13.631 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:33:13.631 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:33:13.702 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:13.705+0000 7fcc55f2d900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:33:13.703 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:13.705+0000 7fcc55f2d900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:33:13.712 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006393.431744_102.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:33:13.431744Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006026.822813_82.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.822813Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:33:13Z"} 2026-03-20T11:33:13.713 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:33:13.727 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-22', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:33:13.727 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-22', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:33:13.727 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-22 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:33:13.793 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:33:13.793 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:33:13.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:13.873+0000 7faa87078900 1 waiting to reach incremental sync.. 2026-03-20T11:33:18.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:18.875+0000 7faa87078900 1 bucket sync caught up with source: 2026-03-20T11:33:18.872 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000002.17.11, , ] 2026-03-20T11:33:18.872 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000002.17.11, , ] 2026-03-20T11:33:18.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:18.875+0000 7faa87078900 0 bucket checkpoint complete 2026-03-20T11:33:18.883 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-22', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:33:18.883 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-22', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:33:18.883 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-22 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:33:18.964 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:33:18.964 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:33:19.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:19.039+0000 7f458f34f900 1 waiting to reach incremental sync.. 2026-03-20T11:33:24.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:24.041+0000 7f458f34f900 1 bucket sync caught up with source: 2026-03-20T11:33:24.038 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000002.22.11, , ] 2026-03-20T11:33:24.038 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000002.22.11, , ] 2026-03-20T11:33:24.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:24.041+0000 7f458f34f900 0 bucket checkpoint complete 2026-03-20T11:33:24.047 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-22 zones={a2, a1} 2026-03-20T11:33:24.056 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:33:24.056 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=Jqadb2CGHazMQ607z7RTQlsUnql.KuU 2026-03-20T11:33:24.059 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:33:24.059 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=Jqadb2CGHazMQ607z7RTQlsUnql.KuU 2026-03-20T11:33:24.059 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:33:24.073 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-22 zones={a2, a1} 2026-03-20T11:33:24.101 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-22', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:33:24.101 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-22', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:33:24.101 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-22 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:33:24.142 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:33:24.142 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:33:24.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:24.217+0000 7f64485cd900 1 waiting for incremental sync to catch up: 2026-03-20T11:33:24.215 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000002.17.11, , ] 2026-03-20T11:33:24.215 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000005.23.12, , ] 2026-03-20T11:33:29.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:29.218+0000 7f64485cd900 1 waiting for incremental sync to catch up: 2026-03-20T11:33:29.216 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000002.17.11, , ] 2026-03-20T11:33:29.216 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000005.23.12, , ] 2026-03-20T11:33:34.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:34.219+0000 7f64485cd900 1 bucket sync caught up with source: 2026-03-20T11:33:34.217 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000006.25.9, , ] 2026-03-20T11:33:34.217 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000005.23.12, , ] 2026-03-20T11:33:34.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:34.219+0000 7f64485cd900 0 bucket checkpoint complete 2026-03-20T11:33:34.226 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-22', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:33:34.226 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-22', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:33:34.226 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-22 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:33:34.307 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:33:34.307 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:33:34.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:34.382+0000 7fedb321e900 1 waiting for incremental sync to catch up: 2026-03-20T11:33:34.379 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000004.26.12, , ] 2026-03-20T11:33:34.379 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000005.29.12, , ] 2026-03-20T11:33:39.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:39.383+0000 7fedb321e900 1 waiting for incremental sync to catch up: 2026-03-20T11:33:39.380 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000004.26.12, , ] 2026-03-20T11:33:39.380 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000005.29.12, , ] 2026-03-20T11:33:44.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:44.384+0000 7fedb321e900 1 waiting for incremental sync to catch up: 2026-03-20T11:33:44.381 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000004.26.12, , ] 2026-03-20T11:33:44.382 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000005.29.12, , ] 2026-03-20T11:33:49.382 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:49.385+0000 7fedb321e900 1 waiting for incremental sync to catch up: 2026-03-20T11:33:49.383 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000004.26.12, , ] 2026-03-20T11:33:49.383 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000005.29.12, , ] 2026-03-20T11:33:54.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:54.386+0000 7fedb321e900 1 bucket sync caught up with source: 2026-03-20T11:33:54.384 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000005.29.12, , ] 2026-03-20T11:33:54.384 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000005.29.12, , ] 2026-03-20T11:33:54.384 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:54.386+0000 7fedb321e900 0 bucket checkpoint complete 2026-03-20T11:33:54.393 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-22 zones={a2, a1} 2026-03-20T11:33:54.399 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:33:54.399 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=Jqadb2CGHazMQ607z7RTQlsUnql.KuU 2026-03-20T11:33:54.399 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=8.mqDXOdol-dg1cQQ7ipAHRS9YeI8sx 2026-03-20T11:33:54.399 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=SbcAMARkY2PUmH.7YwTOFq-uvo9E3EK 2026-03-20T11:33:54.399 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=TwtS1kB9FTDoBtofGwKYBzbPMjh6aFu 2026-03-20T11:33:54.399 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=oArjEDC8Xlv3RqnxNucnYRMmEhpJ3Dv 2026-03-20T11:33:54.402 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:33:54.402 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=Jqadb2CGHazMQ607z7RTQlsUnql.KuU 2026-03-20T11:33:54.402 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=8.mqDXOdol-dg1cQQ7ipAHRS9YeI8sx 2026-03-20T11:33:54.402 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=SbcAMARkY2PUmH.7YwTOFq-uvo9E3EK 2026-03-20T11:33:54.402 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=TwtS1kB9FTDoBtofGwKYBzbPMjh6aFu 2026-03-20T11:33:54.402 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=oArjEDC8Xlv3RqnxNucnYRMmEhpJ3Dv 2026-03-20T11:33:54.402 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:33:54.413 DEBUG:tasks.rgw_multi.zone_rados:both are delete markers, skipping content comparison 2026-03-20T11:33:54.413 DEBUG:tasks.rgw_multi.zone_rados:both are delete markers, skipping content comparison 2026-03-20T11:33:54.413 DEBUG:tasks.rgw_multi.zone_rados:both are delete markers, skipping content comparison 2026-03-20T11:33:54.413 DEBUG:tasks.rgw_multi.zone_rados:both are delete markers, skipping content comparison 2026-03-20T11:33:54.413 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-22 zones={a2, a1} 2026-03-20T11:33:54.415 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_concurrent_delete_markers_incremental_sync ... ok 2026-03-20T11:33:54.515 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:33:54.515 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:33:54.515 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:33:54.550 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:33:54.550 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:33:54.615 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:33:54.634 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774006434.517535_106.1","last_update":"2026-03-20T11:33:54.517535Z"},{"marker":"1_1774006026.822813_82.1","last_update":"2026-03-20T11:27:06.822813Z"}] 2026-03-20T11:33:54.634 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:33:54.635 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:33:54.635 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:33:54.635 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:33:54.712 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:33:54.712 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:33:54.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:54.785+0000 7f9b6af3c900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:33:54.783 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:54.786+0000 7f9b6af3c900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:33:54.794 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006434.517535_106.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:33:54.517535Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006026.822813_82.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.822813Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:33:54Z"} 2026-03-20T11:33:54.794 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:33:54.802 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-23', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:33:54.803 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-23', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:33:54.803 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-23 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:33:54.876 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:33:54.876 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:33:54.954 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:54.957+0000 7fbd3a610900 1 waiting to reach incremental sync.. 2026-03-20T11:33:59.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:59.959+0000 7fbd3a610900 1 bucket sync caught up with source: 2026-03-20T11:33:59.956 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.13.6, , ] 2026-03-20T11:33:59.956 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000001.13.6, , ] 2026-03-20T11:33:59.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:33:59.959+0000 7fbd3a610900 0 bucket checkpoint complete 2026-03-20T11:33:59.969 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-23', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:33:59.969 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-23', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:33:59.969 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-23 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:34:00.049 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:34:00.049 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:34:00.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:34:00.120+0000 7f606df2c900 1 bucket sync caught up with source: 2026-03-20T11:34:00.118 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.26.6, , ] 2026-03-20T11:34:00.118 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000001.26.6, , ] 2026-03-20T11:34:00.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:34:00.120+0000 7f606df2c900 0 bucket checkpoint complete 2026-03-20T11:34:00.127 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-23 zones={a2, a1} 2026-03-20T11:34:00.134 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:34:00.134 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=null 2026-03-20T11:34:00.137 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:34:00.137 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=null 2026-03-20T11:34:00.137 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2026-03-20T11:34:00.151 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-23 zones={a2, a1} 2026-03-20T11:34:00.161 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-23', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:34:00.161 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-23', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:34:00.161 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-23 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:34:00.209 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:34:00.209 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:34:00.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:34:00.282+0000 7f8de5610900 1 waiting for incremental sync to catch up: 2026-03-20T11:34:00.279 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.13.6, , ] 2026-03-20T11:34:00.279 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000003.16.7, , ] 2026-03-20T11:34:05.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:34:05.283+0000 7f8de5610900 1 waiting for incremental sync to catch up: 2026-03-20T11:34:05.280 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.13.6, , ] 2026-03-20T11:34:05.280 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000003.16.7, , ] 2026-03-20T11:34:10.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:34:10.284+0000 7f8de5610900 1 waiting for incremental sync to catch up: 2026-03-20T11:34:10.281 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.13.6, , ] 2026-03-20T11:34:10.281 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000003.16.7, , ] 2026-03-20T11:34:15.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:34:15.285+0000 7f8de5610900 1 waiting for incremental sync to catch up: 2026-03-20T11:34:15.282 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.13.6, , ] 2026-03-20T11:34:15.282 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000003.16.7, , ] 2026-03-20T11:34:20.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:34:20.286+0000 7f8de5610900 1 bucket sync caught up with source: 2026-03-20T11:34:20.283 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000003.16.7, , ] 2026-03-20T11:34:20.283 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000003.16.7, , ] 2026-03-20T11:34:20.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:34:20.286+0000 7f8de5610900 0 bucket checkpoint complete 2026-03-20T11:34:20.293 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-23', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:34:20.293 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-23', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:34:20.293 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-23 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:34:20.374 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:34:20.374 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:34:20.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:34:20.447+0000 7f452ba0f900 1 waiting for incremental sync to catch up: 2026-03-20T11:34:20.444 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.26.6, , ] 2026-03-20T11:34:20.444 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000002.27.13, , ] 2026-03-20T11:34:25.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:34:25.448+0000 7f452ba0f900 1 waiting for incremental sync to catch up: 2026-03-20T11:34:25.445 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.26.6, , ] 2026-03-20T11:34:25.445 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000002.27.13, , ] 2026-03-20T11:34:30.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:34:30.449+0000 7f452ba0f900 1 waiting for incremental sync to catch up: 2026-03-20T11:34:30.446 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.26.6, , ] 2026-03-20T11:34:30.446 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000002.27.13, , ] 2026-03-20T11:34:35.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:34:35.450+0000 7f452ba0f900 1 waiting for incremental sync to catch up: 2026-03-20T11:34:35.447 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.26.6, , ] 2026-03-20T11:34:35.448 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000002.27.13, , ] 2026-03-20T11:34:40.449 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:34:40.451+0000 7f452ba0f900 1 bucket sync caught up with source: 2026-03-20T11:34:40.449 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000002.27.13, , ] 2026-03-20T11:34:40.449 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000002.27.13, , ] 2026-03-20T11:34:40.449 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:34:40.451+0000 7f452ba0f900 0 bucket checkpoint complete 2026-03-20T11:34:40.459 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-23 zones={a2, a1} 2026-03-20T11:34:40.465 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:34:40.465 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=null 2026-03-20T11:34:40.465 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=oiGIvxx7K0d4DtWrW4jatEfVotVanDM 2026-03-20T11:34:40.467 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:34:40.468 DEBUG:tasks.rgw_multi.zone_rados:o=obj, v=null 2026-03-20T11:34:40.468 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_suspended_delete_marker_incremental_sync ... FAIL 2026-03-20T11:34:40.542 INFO:rgw_multi.tests:create bucket zone=a1 name=dipepu-24 2026-03-20T11:34:40.555 INFO:rgw_multi.tests:create bucket zone=a2 name=dipepu-25 2026-03-20T11:34:40.630 INFO:rgw_multi.tests:create bucket zone=b1 name=dipepu-26 2026-03-20T11:34:40.651 INFO:rgw_multi.tests:create bucket zone=b2 name=dipepu-27 2026-03-20T11:34:40.670 INFO:rgw_multi.tests:meta checkpoint 2026-03-20T11:34:40.670 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:40.670 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:40.671 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:34:40.709 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:34:40.710 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:34:40.786 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:34:40.796 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774006480.664599_114.1","last_update":"2026-03-20T11:34:40.664599Z"},{"marker":"1_1774006026.822813_82.1","last_update":"2026-03-20T11:27:06.822813Z"}] 2026-03-20T11:34:40.797 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:34:40.797 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:40.797 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:40.797 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:34:40.876 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:34:40.876 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:34:40.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:34:40.951+0000 7f7baa60f900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:34:40.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:34:40.952+0000 7f7baa60f900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:34:40.958 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006480.664599_114.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:34:40.664599Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006026.822813_82.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.822813Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:34:40Z"} 2026-03-20T11:34:40.958 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:34:40.959 INFO:rgw_multi.tests:starting meta checkpoint for zone=b1 2026-03-20T11:34:40.959 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:40.959 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:40.959 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:34:40.997 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:34:40.997 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:34:41.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:34:41.065+0000 7f9568d86900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:34:41.065 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:34:41.066+0000 7f9568d86900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:34:41.074 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006434.517535_106.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:33:54.517535Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006026.822813_82.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.822813Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:34:41Z"} 2026-03-20T11:34:41.074 WARNING:rgw_multi.tests:zone b1 behind master: shard=2 master=1_1774006480.664599_114.1 target=1_1774006434.517535_106.1 2026-03-20T11:34:46.075 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:46.075 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:46.075 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:34:46.111 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:34:46.111 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:34:46.173 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:34:46.175+0000 7faec2593900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:34:46.174 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:34:46.176+0000 7faec2593900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:34:46.183 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006480.664599_114.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:34:40.664599Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006026.822813_82.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.822813Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:34:46Z"} 2026-03-20T11:34:46.183 INFO:rgw_multi.tests:finish meta checkpoint for zone=b1 2026-03-20T11:34:46.183 INFO:rgw_multi.tests:starting meta checkpoint for zone=b2 2026-03-20T11:34:46.183 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:46.183 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:46.183 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b2 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:34:46.259 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:34:46.259 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:34:46.318 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:34:46.320+0000 7fc965a10900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:34:46.319 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:34:46.321+0000 7fc965a10900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:34:46.329 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006480.664599_114.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:34:40.664599Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006026.822813_82.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.822813Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:34:46Z"} 2026-03-20T11:34:46.329 INFO:rgw_multi.tests:finish meta checkpoint for zone=b2 2026-03-20T11:34:46.540 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_versioning ... ok 2026-03-20T11:34:46.633 INFO:rgw_multi.tests:create bucket zone=a1 name=dipepu-28 2026-03-20T11:34:46.648 INFO:rgw_multi.tests:create bucket zone=a2 name=dipepu-29 2026-03-20T11:34:46.779 INFO:rgw_multi.tests:create bucket zone=b1 name=dipepu-30 2026-03-20T11:34:46.800 INFO:rgw_multi.tests:create bucket zone=b2 name=dipepu-31 2026-03-20T11:34:46.820 INFO:rgw_multi.tests:meta checkpoint 2026-03-20T11:34:46.820 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:46.820 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:46.820 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:34:46.862 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:34:46.862 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:34:46.936 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:34:46.947 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774006486.813293_126.1","last_update":"2026-03-20T11:34:46.813293Z"},{"marker":"1_1774006026.822813_82.1","last_update":"2026-03-20T11:27:06.822813Z"}] 2026-03-20T11:34:46.947 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:34:46.947 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:46.947 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:46.947 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:34:47.025 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:34:47.026 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:34:47.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:34:47.095+0000 7fc5b5928900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:34:47.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:34:47.096+0000 7fc5b5928900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:34:47.101 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006486.813293_126.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:34:46.813293Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006026.822813_82.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.822813Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:34:47Z"} 2026-03-20T11:34:47.101 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:34:47.101 INFO:rgw_multi.tests:starting meta checkpoint for zone=b1 2026-03-20T11:34:47.101 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:47.101 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:47.101 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:34:47.138 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:34:47.138 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:34:47.202 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:34:47.203+0000 7f7dfbf26900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:34:47.202 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:34:47.204+0000 7f7dfbf26900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:34:47.211 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006480.664599_114.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:34:40.664599Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006026.822813_82.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.822813Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:34:47Z"} 2026-03-20T11:34:47.211 WARNING:rgw_multi.tests:zone b1 behind master: shard=2 master=1_1774006486.813293_126.1 target=1_1774006480.664599_114.1 2026-03-20T11:34:52.212 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:52.212 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:52.212 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:34:52.251 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:34:52.251 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:34:52.316 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:34:52.317+0000 7f83d18fa900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:34:52.317 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:34:52.318+0000 7f83d18fa900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:34:52.324 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006486.813293_126.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:34:46.813293Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006026.822813_82.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.822813Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:34:52Z"} 2026-03-20T11:34:52.325 INFO:rgw_multi.tests:finish meta checkpoint for zone=b1 2026-03-20T11:34:52.325 INFO:rgw_multi.tests:starting meta checkpoint for zone=b2 2026-03-20T11:34:52.325 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:52.325 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:52.325 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b2 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:34:52.404 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:34:52.404 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:34:52.474 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:34:52.476+0000 7f6e51e1c900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:34:52.475 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:34:52.477+0000 7f6e51e1c900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:34:52.484 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006486.813293_126.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:34:46.813293Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006026.822813_82.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.822813Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:34:52Z"} 2026-03-20T11:34:52.484 INFO:rgw_multi.tests:finish meta checkpoint for zone=b2 2026-03-20T11:34:52.704 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_acl ... ok 2026-03-20T11:34:52.759 INFO:rgw_multi.tests:create bucket zone=a1 name=dipepu-32 2026-03-20T11:34:52.774 INFO:rgw_multi.tests:create bucket zone=a2 name=dipepu-33 2026-03-20T11:34:52.914 INFO:rgw_multi.tests:create bucket zone=b1 name=dipepu-34 2026-03-20T11:34:52.943 INFO:rgw_multi.tests:create bucket zone=b2 name=dipepu-35 2026-03-20T11:34:52.965 INFO:rgw_multi.tests:meta checkpoint 2026-03-20T11:34:52.965 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:52.965 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:52.965 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:34:53.002 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:34:53.002 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:34:53.073 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:34:53.087 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774006492.957699_138.1","last_update":"2026-03-20T11:34:52.957699Z"},{"marker":"1_1774006026.822813_82.1","last_update":"2026-03-20T11:27:06.822813Z"}] 2026-03-20T11:34:53.087 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:34:53.087 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:53.087 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:53.087 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:34:53.164 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:34:53.164 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:34:53.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:34:53.242+0000 7fb4a3472900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:34:53.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:34:53.243+0000 7fb4a3472900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:34:53.251 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006492.957699_138.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:34:52.957699Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006026.822813_82.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.822813Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:34:53Z"} 2026-03-20T11:34:53.251 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:34:53.251 INFO:rgw_multi.tests:starting meta checkpoint for zone=b1 2026-03-20T11:34:53.251 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:53.251 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:53.251 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:34:53.293 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:34:53.293 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:34:53.361 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:34:53.363+0000 7f11b2930900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:34:53.362 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:34:53.364+0000 7f11b2930900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:34:53.372 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006486.813293_126.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:34:46.813293Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006026.822813_82.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.822813Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:34:53Z"} 2026-03-20T11:34:53.372 WARNING:rgw_multi.tests:zone b1 behind master: shard=2 master=1_1774006492.957699_138.1 target=1_1774006486.813293_126.1 2026-03-20T11:34:58.373 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:58.373 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:58.373 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:34:58.415 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:34:58.415 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:34:58.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:34:58.489+0000 7fb04e212900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:34:58.488 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:34:58.490+0000 7fb04e212900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:34:58.499 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006492.957699_138.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:34:52.957699Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006026.822813_82.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.822813Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:34:58Z"} 2026-03-20T11:34:58.499 INFO:rgw_multi.tests:finish meta checkpoint for zone=b1 2026-03-20T11:34:58.499 INFO:rgw_multi.tests:starting meta checkpoint for zone=b2 2026-03-20T11:34:58.499 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:58.499 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:34:58.499 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b2 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:34:58.578 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:34:58.578 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:34:58.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:34:58.654+0000 7f212e530900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:34:58.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:34:58.655+0000 7f212e530900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:34:58.663 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006492.957699_138.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:34:52.957699Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006026.822813_82.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.822813Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:34:58Z"} 2026-03-20T11:34:58.664 INFO:rgw_multi.tests:finish meta checkpoint for zone=b2 2026-03-20T11:35:01.880 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_cors ... ok 2026-03-20T11:35:01.989 INFO:rgw_multi.tests:create bucket zone=a1 name=dipepu-36 2026-03-20T11:35:02.005 INFO:rgw_multi.tests:create bucket zone=a2 name=dipepu-37 2026-03-20T11:35:02.027 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:02.027 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:02.027 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:35:02.065 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:02.066 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:02.141 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:35:02.152 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774006502.020991_146.1","last_update":"2026-03-20T11:35:02.020991Z"},{"marker":"1_1774006026.822813_82.1","last_update":"2026-03-20T11:27:06.822813Z"}] 2026-03-20T11:35:02.152 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:35:02.152 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:02.152 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:02.152 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:35:02.226 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:02.226 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:02.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:35:02.298+0000 7f99fca10900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:35:02.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:35:02.299+0000 7f99fca10900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:35:02.306 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006502.020991_146.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:35:02.020991Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006026.822813_82.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.822813Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:35:02Z"} 2026-03-20T11:35:02.306 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:35:02.429 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_delete_notempty ... ok 2026-03-20T11:35:02.429 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_multi_period_incremental_sync ... SKIP: test_multi_period_incremental_sync skipped. Requires 3 or more zones in master zonegroup. 2026-03-20T11:35:02.472 INFO:rgw_multi.tests:create bucket zone=a1 name=dipepu-38 2026-03-20T11:35:02.485 INFO:rgw_multi.tests:create bucket zone=a2 name=dipepu-39 2026-03-20T11:35:02.524 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:02.524 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:02.524 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:35:02.564 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:02.565 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:02.634 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:35:02.644 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774006502.501814_150.1","last_update":"2026-03-20T11:35:02.501814Z"},{"marker":"1_1774006026.822813_82.1","last_update":"2026-03-20T11:27:06.822813Z"}] 2026-03-20T11:35:02.644 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:35:02.644 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:02.644 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:02.644 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:35:02.719 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:02.720 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:02.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:35:02.808+0000 7fef37fcd900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:35:02.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:35:02.809+0000 7fef37fcd900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:35:02.817 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006502.501814_150.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:35:02.501814Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006026.822813_82.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.822813Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:35:02Z"} 2026-03-20T11:35:02.818 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:35:02.818 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:02.818 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:02.818 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:35:02.897 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:02.897 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:02.997 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"00000000000000000000:00000000000000003357","last_update":"2026-03-20T11:28:44.807706Z"},{"marker":"00000000000000000000:00000000000000008989","last_update":"2026-03-20T11:35:02.517764Z"},{"marker":"00000000000000000000:00000000000000006319","last_update":"2026-03-20T11:31:18.835901Z"},{"marker":"00000000000000000000:00000000000000004056","last_update":"2026-03-20T11:35:02.316304Z"}] 2026-03-20T11:35:02.997 INFO:rgw_multi.tests:starting data checkpoint for target_zone=a2 source_zone=a1 2026-03-20T11:35:02.997 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:02.997 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:02.998 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --source-zone a1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:35:03.074 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:03.075 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:03.149 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"instance_id":15702199979187085538},"markers":[{"key":0,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000003357","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:28:44.807706Z"}},{"key":1,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000008848","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:34:14.861863Z"}},{"key":2,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000006319","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:31:18.835901Z"}},{"key":3,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000003914","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:28:44.813049Z"}}]},"full_sync":{"total":0,"complete":0},"current_time":"2026-03-20T11:35:03Z"} 2026-03-20T11:35:03.149 WARNING:rgw_multi.tests:data of zone a2 behind zone a1: shard=1 master=00000000000000000000:00000000000000008989 target=00000000000000000000:00000000000000008848, shard=3 master=00000000000000000000:00000000000000004056 target=00000000000000000000:00000000000000003914 2026-03-20T11:35:08.150 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:08.150 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:08.150 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --source-zone a1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:35:08.189 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:08.189 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:08.266 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"instance_id":15702199979187085538},"markers":[{"key":0,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000003357","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:28:44.807706Z"}},{"key":1,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000009130","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:35:04.273160Z"}},{"key":2,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000006319","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:31:18.835901Z"}},{"key":3,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000004198","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:35:04.193357Z"}}]},"full_sync":{"total":0,"complete":0},"current_time":"2026-03-20T11:35:08Z"} 2026-03-20T11:35:08.266 INFO:rgw_multi.tests:finished data checkpoint for target_zone=a2 source_zone=a1 2026-03-20T11:35:08.266 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:08.267 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:08.267 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:35:08.345 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:08.345 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:08.428 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"00000000000000000000:00000000000000003639","last_update":"2026-03-20T11:28:20.439213Z"},{"marker":"00000000000000000000:00000000000000009694","last_update":"2026-03-20T11:35:07.271155Z"},{"marker":"00000000000000000000:00000000000000006742","last_update":"2026-03-20T11:31:16.629414Z"},{"marker":"00000000000000000000:00000000000000004622","last_update":"2026-03-20T11:35:07.066474Z"}] 2026-03-20T11:35:08.428 INFO:rgw_multi.tests:starting data checkpoint for target_zone=a1 source_zone=a2 2026-03-20T11:35:08.428 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:08.428 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:08.428 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --source-zone a2 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:35:08.507 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:08.507 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:08.589 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"instance_id":5866003269306126155},"markers":[{"key":0,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000003639","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:28:20.439213Z"}},{"key":1,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000009553","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:35:02.525870Z"}},{"key":2,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000006742","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:31:16.629414Z"}},{"key":3,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000004480","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:35:02.373811Z"}}]},"full_sync":{"total":0,"complete":0},"current_time":"2026-03-20T11:35:08Z"} 2026-03-20T11:35:08.589 WARNING:rgw_multi.tests:data of zone a1 behind zone a2: shard=1 master=00000000000000000000:00000000000000009694 target=00000000000000000000:00000000000000009553, shard=3 master=00000000000000000000:00000000000000004622 target=00000000000000000000:00000000000000004480 2026-03-20T11:35:13.590 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:13.590 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:13.590 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --source-zone a2 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:35:13.629 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:13.629 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:13.710 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"instance_id":5866003269306126155},"markers":[{"key":0,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000003639","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:28:20.439213Z"}},{"key":1,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000009694","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:35:07.271155Z"}},{"key":2,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000006742","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:31:16.629414Z"}},{"key":3,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000004622","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:35:07.066474Z"}}]},"full_sync":{"total":0,"complete":0},"current_time":"2026-03-20T11:35:13Z"} 2026-03-20T11:35:13.710 INFO:rgw_multi.tests:finished data checkpoint for target_zone=a1 source_zone=a2 2026-03-20T11:35:13.710 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-39', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:35:13.710 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-39', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:35:13.710 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-39 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:35:13.793 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:13.793 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:13.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:35:13.872+0000 7fc13fc10900 1 bucket sync caught up with source: 2026-03-20T11:35:13.869 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.10.6, , ] 2026-03-20T11:35:13.869 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000001.10.6, , ] 2026-03-20T11:35:13.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:35:13.872+0000 7fc13fc10900 0 bucket checkpoint complete 2026-03-20T11:35:13.879 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-39', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:35:13.879 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-39', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:35:13.879 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-39 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:35:13.919 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:13.919 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:13.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:35:13.996+0000 7f5c85a1c900 1 bucket sync caught up with source: 2026-03-20T11:35:13.993 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , , , , 00000000001.18.6, , ] 2026-03-20T11:35:13.993 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , , , , 00000000001.18.6, , ] 2026-03-20T11:35:13.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:35:13.996+0000 7f5c85a1c900 0 bucket checkpoint complete 2026-03-20T11:35:14.003 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-39 zones={a2, a1} 2026-03-20T11:35:14.075 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:35:14.075 DEBUG:tasks.rgw_multi.zone_rados:o=key, v=null 2026-03-20T11:35:14.079 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:35:14.079 DEBUG:tasks.rgw_multi.zone_rados:o=key, v=null 2026-03-20T11:35:14.079 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=key 2026-03-20T11:35:14.135 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-39 zones={a2, a1} 2026-03-20T11:35:19.135 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:19.136 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:19.136 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:35:19.171 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:19.171 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:19.255 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"00000000000000000000:00000000000000003357","last_update":"2026-03-20T11:28:44.807706Z"},{"marker":"00000000000000000000:00000000000000009130","last_update":"2026-03-20T11:35:04.273160Z"},{"marker":"00000000000000000000:00000000000000006319","last_update":"2026-03-20T11:31:18.835901Z"},{"marker":"00000000000000000000:00000000000000004198","last_update":"2026-03-20T11:35:04.193357Z"}] 2026-03-20T11:35:19.255 INFO:rgw_multi.tests:starting data checkpoint for target_zone=a2 source_zone=a1 2026-03-20T11:35:19.255 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:19.255 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:19.255 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --source-zone a1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:35:19.333 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:19.334 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:19.412 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"instance_id":15702199979187085538},"markers":[{"key":0,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000003357","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:28:44.807706Z"}},{"key":1,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000009130","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:35:04.273160Z"}},{"key":2,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000006319","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:31:18.835901Z"}},{"key":3,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000004198","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:35:04.193357Z"}}]},"full_sync":{"total":0,"complete":0},"current_time":"2026-03-20T11:35:19Z"} 2026-03-20T11:35:19.412 INFO:rgw_multi.tests:finished data checkpoint for target_zone=a2 source_zone=a1 2026-03-20T11:35:19.412 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:19.413 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:19.413 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:35:19.489 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:19.489 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:19.576 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"00000000000000000000:00000000000000003639","last_update":"2026-03-20T11:28:20.439213Z"},{"marker":"00000000000000000000:00000000000000009694","last_update":"2026-03-20T11:35:07.271155Z"},{"marker":"00000000000000000000:00000000000000006742","last_update":"2026-03-20T11:31:16.629414Z"},{"marker":"00000000000000000000:00000000000000004622","last_update":"2026-03-20T11:35:07.066474Z"}] 2026-03-20T11:35:19.577 INFO:rgw_multi.tests:starting data checkpoint for target_zone=a1 source_zone=a2 2026-03-20T11:35:19.577 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:19.577 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:19.577 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --source-zone a2 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:35:19.659 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:19.660 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:19.739 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"instance_id":5866003269306126155},"markers":[{"key":0,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000003639","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:28:20.439213Z"}},{"key":1,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000009694","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:35:07.271155Z"}},{"key":2,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000006742","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:31:16.629414Z"}},{"key":3,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000004622","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:35:07.066474Z"}}]},"full_sync":{"total":0,"complete":0},"current_time":"2026-03-20T11:35:19Z"} 2026-03-20T11:35:19.740 INFO:rgw_multi.tests:finished data checkpoint for target_zone=a1 source_zone=a2 2026-03-20T11:35:19.740 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:19.740 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:19.740 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:35:19.813 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:19.813 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:19.894 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"00000000000000000000:00000000000000003357","last_update":"2026-03-20T11:28:44.807706Z"},{"marker":"00000000000000000000:00000000000000009130","last_update":"2026-03-20T11:35:04.273160Z"},{"marker":"00000000000000000000:00000000000000006319","last_update":"2026-03-20T11:31:18.835901Z"},{"marker":"00000000000000000000:00000000000000004198","last_update":"2026-03-20T11:35:04.193357Z"}] 2026-03-20T11:35:19.894 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'autotrim', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:35:19.894 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'autotrim', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:35:19.894 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog autotrim --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:35:19.967 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:19.967 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:20.050 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'list', '--shard-id', '0', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:20.050 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'list', '--shard-id', '0', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:20.050 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog list --shard-id 0 --max-entries 1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:35:20.129 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:20.129 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:20.206 INFO:teuthology.orchestra.run.vm02.stdout:[] 2026-03-20T11:35:20.206 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'list', '--shard-id', '1', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:20.206 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'list', '--shard-id', '1', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:20.206 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog list --shard-id 1 --max-entries 1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:35:20.282 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:20.282 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:20.358 INFO:teuthology.orchestra.run.vm02.stdout:[] 2026-03-20T11:35:20.358 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'list', '--shard-id', '2', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:20.358 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'list', '--shard-id', '2', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:20.358 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog list --shard-id 2 --max-entries 1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:35:20.432 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:20.432 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:20.506 INFO:teuthology.orchestra.run.vm02.stdout:[] 2026-03-20T11:35:20.506 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'list', '--shard-id', '3', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:20.506 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'list', '--shard-id', '3', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:20.506 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog list --shard-id 3 --max-entries 1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:35:20.580 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:20.580 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:20.653 INFO:teuthology.orchestra.run.vm02.stdout:[] 2026-03-20T11:35:20.653 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:20.653 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:20.653 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:35:20.725 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:20.725 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:20.807 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"00000000000000000000:00000000000000003357","last_update":"2026-03-20T11:28:44.807706Z"},{"marker":"00000000000000000000:00000000000000009130","last_update":"2026-03-20T11:35:04.273160Z"},{"marker":"00000000000000000000:00000000000000006319","last_update":"2026-03-20T11:31:18.835901Z"},{"marker":"00000000000000000000:00000000000000004198","last_update":"2026-03-20T11:35:04.193357Z"}] 2026-03-20T11:35:20.807 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'autotrim', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:35:20.808 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'autotrim', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:35:20.808 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog autotrim --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:35:20.880 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:20.881 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:20.965 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'list', '--shard-id', '0', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:20.965 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'list', '--shard-id', '0', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:20.965 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog list --shard-id 0 --max-entries 1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:35:21.039 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:21.039 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:21.115 INFO:teuthology.orchestra.run.vm02.stdout:[] 2026-03-20T11:35:21.115 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'list', '--shard-id', '1', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:21.115 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'list', '--shard-id', '1', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:21.115 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog list --shard-id 1 --max-entries 1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:35:21.191 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:21.191 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:21.265 INFO:teuthology.orchestra.run.vm02.stdout:[] 2026-03-20T11:35:21.265 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'list', '--shard-id', '2', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:21.265 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'list', '--shard-id', '2', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:21.265 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog list --shard-id 2 --max-entries 1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:35:21.339 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:21.339 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:21.413 INFO:teuthology.orchestra.run.vm02.stdout:[] 2026-03-20T11:35:21.413 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'list', '--shard-id', '3', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:21.413 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'list', '--shard-id', '3', '--max-entries', '1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:35:21.413 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog list --shard-id 3 --max-entries 1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:35:21.487 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:21.487 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:21.560 INFO:teuthology.orchestra.run.vm02.stdout:[] 2026-03-20T11:35:21.562 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_datalog_autotrim ... ok 2026-03-20T11:35:21.606 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zone', 'modify', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--sync-from-all=false', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:35:21.606 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zone', 'modify', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--sync-from-all=false', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:35:21.606 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zone modify --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --sync-from-all=false --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:35:21.640 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:21.640 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:21.660 INFO:teuthology.orchestra.run.vm02.stdout:{"id":"51055576-6662-409d-8abb-ecaeefd3dd80","name":"a2","domain_root":"a2.rgw.meta:root","control_pool":"a2.rgw.control","dedup_pool":"a2.rgw.dedup","gc_pool":"a2.rgw.log:gc","lc_pool":"a2.rgw.log:lc","log_pool":"a2.rgw.log","intent_log_pool":"a2.rgw.log:intent","usage_log_pool":"a2.rgw.log:usage","roles_pool":"a2.rgw.meta:roles","reshard_pool":"a2.rgw.log:reshard","user_keys_pool":"a2.rgw.meta:users.keys","user_email_pool":"a2.rgw.meta:users.email","user_swift_pool":"a2.rgw.meta:users.swift","user_uid_pool":"a2.rgw.meta:users.uid","otp_pool":"a2.rgw.otp","notif_pool":"a2.rgw.log:notif","topics_pool":"a2.rgw.meta:topics","account_pool":"a2.rgw.meta:accounts","group_pool":"a2.rgw.meta:groups","system_key":{"access_key":"GRL33KL1ULN5795C","secret_key":"FPGvuVSJOG1341zbJPfw7ch710f3izIf"},"placement_pools":[{"key":"default-placement","val":{"index_pool":"a2.rgw.buckets.index","storage_classes":{"STANDARD":{"data_pool":"a2.rgw.buckets.data","compression_type":"random"}},"data_extra_pool":"a2.rgw.buckets.non-ec","index_type":0,"inline_data":true}}],"realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","restore_pool":"a2.rgw.log:restore"} 2026-03-20T11:35:21.660 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['period', 'update', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:35:21.660 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'period', 'update', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:35:21.660 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 period update --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --commit --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:35:21.733 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:35:21.734 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:35:21.810 INFO:teuthology.orchestra.run.vm02.stderr:Sending period to new master zone 4b634ebf-58f5-45bc-839a-cb4b425e773a 2026-03-20T11:35:21.848 INFO:teuthology.orchestra.run.vm02.stdout:{"id":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","epoch":5,"predecessor_uuid":"5d4b2b15-cd70-4a63-87c8-e59369f9ab35","sync_status":[],"period_map":{"id":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","zonegroups":[{"id":"a","name":"a","api_name":"a","is_master":true,"endpoints":["http://vm02.local:8000"],"hostnames":[],"hostnames_s3website":[],"master_zone":"4b634ebf-58f5-45bc-839a-cb4b425e773a","zones":[{"id":"4b634ebf-58f5-45bc-839a-cb4b425e773a","name":"a1","endpoints":["http://vm02.local:8000"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]},{"id":"51055576-6662-409d-8abb-ecaeefd3dd80","name":"a2","endpoints":["http://vm05.local:8001"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":false,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]}],"placement_targets":[{"name":"default-placement","tags":[],"storage_classes":["STANDARD"]}],"default_placement":"default-placement","realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","sync_policy":{"groups":[]},"enabled_features":["notification_v2","resharding"]},{"id":"b","name":"b","api_name":"b","is_master":false,"endpoints":["http://vm07.local:8000"],"hostnames":[],"hostnames_s3website":[],"master_zone":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","zones":[{"id":"cd52b229-0d4a-43bf-b4ba-156aa19e7aef","name":"b2","endpoints":["http://vm08.local:8001"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]},{"id":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","name":"b1","endpoints":["http://vm07.local:8000"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]}],"placement_targets":[{"name":"default-placement","tags":[],"storage_classes":["STANDARD"]}],"default_placement":"default-placement","realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","sync_policy":{"groups":[]},"enabled_features":["notification_v2","resharding"]}],"short_zone_ids":[{"key":"4b634ebf-58f5-45bc-839a-cb4b425e773a","val":162505008},{"key":"51055576-6662-409d-8abb-ecaeefd3dd80","val":2776933003},{"key":"cd52b229-0d4a-43bf-b4ba-156aa19e7aef","val":2686426109},{"key":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","val":114662526}]},"master_zonegroup":"a","master_zone":"4b634ebf-58f5-45bc-839a-cb4b425e773a","period_config":{"bucket_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false},"bucket_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false},"anonymous_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false}},"realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","realm_epoch":2} 2026-03-20T11:35:21.848 INFO:rgw_multi.tests:Set sync_from_all flag on zone a2 to false 2026-03-20T11:36:51.848 INFO:rgw_multi.tests:create bucket zone=a1 name=dipepu-40 2026-03-20T11:36:51.873 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:36:51.873 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:36:51.873 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:36:51.913 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:36:51.913 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:36:51.987 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:36:51.998 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774006611.864443_171.1","last_update":"2026-03-20T11:36:51.864443Z"},{"marker":"1_1774006611.861570_128.1","last_update":"2026-03-20T11:36:51.861570Z"}] 2026-03-20T11:36:51.999 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:36:51.999 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:36:51.999 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:36:51.999 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:36:52.081 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:36:52.081 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:36:52.162 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:36:52.164+0000 7f7268410900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:36:52.163 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:36:52.165+0000 7f7268410900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:36:52.173 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006611.864443_171.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.864443Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006611.861570_128.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.861570Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:36:52Z"} 2026-03-20T11:36:52.173 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:36:52.178 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zone', 'modify', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--redirect-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:36:52.178 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zone', 'modify', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--redirect-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:36:52.178 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zone modify --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --redirect-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:36:52.249 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:36:52.249 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:36:52.271 INFO:teuthology.orchestra.run.vm02.stdout:{"id":"51055576-6662-409d-8abb-ecaeefd3dd80","name":"a2","domain_root":"a2.rgw.meta:root","control_pool":"a2.rgw.control","dedup_pool":"a2.rgw.dedup","gc_pool":"a2.rgw.log:gc","lc_pool":"a2.rgw.log:lc","log_pool":"a2.rgw.log","intent_log_pool":"a2.rgw.log:intent","usage_log_pool":"a2.rgw.log:usage","roles_pool":"a2.rgw.meta:roles","reshard_pool":"a2.rgw.log:reshard","user_keys_pool":"a2.rgw.meta:users.keys","user_email_pool":"a2.rgw.meta:users.email","user_swift_pool":"a2.rgw.meta:users.swift","user_uid_pool":"a2.rgw.meta:users.uid","otp_pool":"a2.rgw.otp","notif_pool":"a2.rgw.log:notif","topics_pool":"a2.rgw.meta:topics","account_pool":"a2.rgw.meta:accounts","group_pool":"a2.rgw.meta:groups","system_key":{"access_key":"GRL33KL1ULN5795C","secret_key":"FPGvuVSJOG1341zbJPfw7ch710f3izIf"},"placement_pools":[{"key":"default-placement","val":{"index_pool":"a2.rgw.buckets.index","storage_classes":{"STANDARD":{"data_pool":"a2.rgw.buckets.data","compression_type":"random"}},"data_extra_pool":"a2.rgw.buckets.non-ec","index_type":0,"inline_data":true}}],"realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","restore_pool":"a2.rgw.log:restore"} 2026-03-20T11:36:52.271 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['period', 'update', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:36:52.271 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'period', 'update', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:36:52.272 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 period update --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --commit --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:36:52.347 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:36:52.347 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:36:52.430 INFO:teuthology.orchestra.run.vm02.stderr:Sending period to new master zone 4b634ebf-58f5-45bc-839a-cb4b425e773a 2026-03-20T11:36:52.468 INFO:teuthology.orchestra.run.vm02.stdout:{"id":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","epoch":6,"predecessor_uuid":"5d4b2b15-cd70-4a63-87c8-e59369f9ab35","sync_status":[],"period_map":{"id":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","zonegroups":[{"id":"a","name":"a","api_name":"a","is_master":true,"endpoints":["http://vm02.local:8000"],"hostnames":[],"hostnames_s3website":[],"master_zone":"4b634ebf-58f5-45bc-839a-cb4b425e773a","zones":[{"id":"4b634ebf-58f5-45bc-839a-cb4b425e773a","name":"a1","endpoints":["http://vm02.local:8000"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]},{"id":"51055576-6662-409d-8abb-ecaeefd3dd80","name":"a2","endpoints":["http://vm05.local:8001"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":false,"sync_from":[],"redirect_zone":"4b634ebf-58f5-45bc-839a-cb4b425e773a","supported_features":["compress-encrypted","notification_v2","resharding"]}],"placement_targets":[{"name":"default-placement","tags":[],"storage_classes":["STANDARD"]}],"default_placement":"default-placement","realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","sync_policy":{"groups":[]},"enabled_features":["notification_v2","resharding"]},{"id":"b","name":"b","api_name":"b","is_master":false,"endpoints":["http://vm07.local:8000"],"hostnames":[],"hostnames_s3website":[],"master_zone":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","zones":[{"id":"cd52b229-0d4a-43bf-b4ba-156aa19e7aef","name":"b2","endpoints":["http://vm08.local:8001"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]},{"id":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","name":"b1","endpoints":["http://vm07.local:8000"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]}],"placement_targets":[{"name":"default-placement","tags":[],"storage_classes":["STANDARD"]}],"default_placement":"default-placement","realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","sync_policy":{"groups":[]},"enabled_features":["notification_v2","resharding"]}],"short_zone_ids":[{"key":"4b634ebf-58f5-45bc-839a-cb4b425e773a","val":162505008},{"key":"51055576-6662-409d-8abb-ecaeefd3dd80","val":2776933003},{"key":"cd52b229-0d4a-43bf-b4ba-156aa19e7aef","val":2686426109},{"key":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","val":114662526}]},"master_zonegroup":"a","master_zone":"4b634ebf-58f5-45bc-839a-cb4b425e773a","period_config":{"bucket_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false},"bucket_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false},"anonymous_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false}},"realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","realm_epoch":2} 2026-03-20T11:36:52.468 INFO:rgw_multi.tests:Set redirect_zone zone a2 to "4b634ebf-58f5-45bc-839a-cb4b425e773a" 2026-03-20T11:38:22.476 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zone', 'modify', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--sync-from-all=true', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:38:22.476 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zone', 'modify', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--sync-from-all=true', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:38:22.476 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zone modify --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --sync-from-all=true --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:38:22.514 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:38:22.514 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:38:22.538 INFO:teuthology.orchestra.run.vm02.stdout:{"id":"51055576-6662-409d-8abb-ecaeefd3dd80","name":"a2","domain_root":"a2.rgw.meta:root","control_pool":"a2.rgw.control","dedup_pool":"a2.rgw.dedup","gc_pool":"a2.rgw.log:gc","lc_pool":"a2.rgw.log:lc","log_pool":"a2.rgw.log","intent_log_pool":"a2.rgw.log:intent","usage_log_pool":"a2.rgw.log:usage","roles_pool":"a2.rgw.meta:roles","reshard_pool":"a2.rgw.log:reshard","user_keys_pool":"a2.rgw.meta:users.keys","user_email_pool":"a2.rgw.meta:users.email","user_swift_pool":"a2.rgw.meta:users.swift","user_uid_pool":"a2.rgw.meta:users.uid","otp_pool":"a2.rgw.otp","notif_pool":"a2.rgw.log:notif","topics_pool":"a2.rgw.meta:topics","account_pool":"a2.rgw.meta:accounts","group_pool":"a2.rgw.meta:groups","system_key":{"access_key":"GRL33KL1ULN5795C","secret_key":"FPGvuVSJOG1341zbJPfw7ch710f3izIf"},"placement_pools":[{"key":"default-placement","val":{"index_pool":"a2.rgw.buckets.index","storage_classes":{"STANDARD":{"data_pool":"a2.rgw.buckets.data","compression_type":"random"}},"data_extra_pool":"a2.rgw.buckets.non-ec","index_type":0,"inline_data":true}}],"realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","restore_pool":"a2.rgw.log:restore"} 2026-03-20T11:38:22.538 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['period', 'update', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:38:22.538 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'period', 'update', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:38:22.538 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 period update --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --commit --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:38:22.616 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:38:22.617 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:38:22.705 INFO:teuthology.orchestra.run.vm02.stderr:Sending period to new master zone 4b634ebf-58f5-45bc-839a-cb4b425e773a 2026-03-20T11:38:22.743 INFO:teuthology.orchestra.run.vm02.stdout:{"id":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","epoch":7,"predecessor_uuid":"5d4b2b15-cd70-4a63-87c8-e59369f9ab35","sync_status":[],"period_map":{"id":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","zonegroups":[{"id":"a","name":"a","api_name":"a","is_master":true,"endpoints":["http://vm02.local:8000"],"hostnames":[],"hostnames_s3website":[],"master_zone":"4b634ebf-58f5-45bc-839a-cb4b425e773a","zones":[{"id":"4b634ebf-58f5-45bc-839a-cb4b425e773a","name":"a1","endpoints":["http://vm02.local:8000"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]},{"id":"51055576-6662-409d-8abb-ecaeefd3dd80","name":"a2","endpoints":["http://vm05.local:8001"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"4b634ebf-58f5-45bc-839a-cb4b425e773a","supported_features":["compress-encrypted","notification_v2","resharding"]}],"placement_targets":[{"name":"default-placement","tags":[],"storage_classes":["STANDARD"]}],"default_placement":"default-placement","realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","sync_policy":{"groups":[]},"enabled_features":["notification_v2","resharding"]},{"id":"b","name":"b","api_name":"b","is_master":false,"endpoints":["http://vm07.local:8000"],"hostnames":[],"hostnames_s3website":[],"master_zone":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","zones":[{"id":"cd52b229-0d4a-43bf-b4ba-156aa19e7aef","name":"b2","endpoints":["http://vm08.local:8001"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]},{"id":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","name":"b1","endpoints":["http://vm07.local:8000"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]}],"placement_targets":[{"name":"default-placement","tags":[],"storage_classes":["STANDARD"]}],"default_placement":"default-placement","realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","sync_policy":{"groups":[]},"enabled_features":["notification_v2","resharding"]}],"short_zone_ids":[{"key":"4b634ebf-58f5-45bc-839a-cb4b425e773a","val":162505008},{"key":"51055576-6662-409d-8abb-ecaeefd3dd80","val":2776933003},{"key":"cd52b229-0d4a-43bf-b4ba-156aa19e7aef","val":2686426109},{"key":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","val":114662526}]},"master_zonegroup":"a","master_zone":"4b634ebf-58f5-45bc-839a-cb4b425e773a","period_config":{"bucket_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false},"bucket_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false},"anonymous_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false}},"realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","realm_epoch":2} 2026-03-20T11:38:22.743 INFO:rgw_multi.tests:Set sync_from_all flag on zone a2 to true 2026-03-20T11:38:43.951 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:38:43.952+0000 7fcfa899f640 -1 WARNING: The OSD cluster is overloaded and struggling to complete ops. You need more capacity to serve this level of demand. 2026-03-20T11:38:44.161 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:38:44.161+0000 7f3eed1c0640 -1 req 7023226894320118835 0.004000004s :post_period failed to decode period 2026-03-20T11:39:52.744 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zone', 'modify', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--redirect-zone=', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:39:52.744 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zone', 'modify', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--redirect-zone=', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:39:52.745 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zone modify --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --redirect-zone= --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:39:52.780 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:39:52.780 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:39:52.804 INFO:teuthology.orchestra.run.vm02.stdout:{"id":"51055576-6662-409d-8abb-ecaeefd3dd80","name":"a2","domain_root":"a2.rgw.meta:root","control_pool":"a2.rgw.control","dedup_pool":"a2.rgw.dedup","gc_pool":"a2.rgw.log:gc","lc_pool":"a2.rgw.log:lc","log_pool":"a2.rgw.log","intent_log_pool":"a2.rgw.log:intent","usage_log_pool":"a2.rgw.log:usage","roles_pool":"a2.rgw.meta:roles","reshard_pool":"a2.rgw.log:reshard","user_keys_pool":"a2.rgw.meta:users.keys","user_email_pool":"a2.rgw.meta:users.email","user_swift_pool":"a2.rgw.meta:users.swift","user_uid_pool":"a2.rgw.meta:users.uid","otp_pool":"a2.rgw.otp","notif_pool":"a2.rgw.log:notif","topics_pool":"a2.rgw.meta:topics","account_pool":"a2.rgw.meta:accounts","group_pool":"a2.rgw.meta:groups","system_key":{"access_key":"GRL33KL1ULN5795C","secret_key":"FPGvuVSJOG1341zbJPfw7ch710f3izIf"},"placement_pools":[{"key":"default-placement","val":{"index_pool":"a2.rgw.buckets.index","storage_classes":{"STANDARD":{"data_pool":"a2.rgw.buckets.data","compression_type":"random"}},"data_extra_pool":"a2.rgw.buckets.non-ec","index_type":0,"inline_data":true}}],"realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","restore_pool":"a2.rgw.log:restore"} 2026-03-20T11:39:52.804 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['period', 'update', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:39:52.804 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'period', 'update', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:39:52.804 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 period update --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --commit --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:39:52.882 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:39:52.882 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:39:53.066 INFO:teuthology.orchestra.run.vm02.stderr:Sending period to new master zone 4b634ebf-58f5-45bc-839a-cb4b425e773a 2026-03-20T11:39:53.110 INFO:teuthology.orchestra.run.vm02.stdout:{"id":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","epoch":8,"predecessor_uuid":"5d4b2b15-cd70-4a63-87c8-e59369f9ab35","sync_status":[],"period_map":{"id":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","zonegroups":[{"id":"a","name":"a","api_name":"a","is_master":true,"endpoints":["http://vm02.local:8000"],"hostnames":[],"hostnames_s3website":[],"master_zone":"4b634ebf-58f5-45bc-839a-cb4b425e773a","zones":[{"id":"4b634ebf-58f5-45bc-839a-cb4b425e773a","name":"a1","endpoints":["http://vm02.local:8000"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]},{"id":"51055576-6662-409d-8abb-ecaeefd3dd80","name":"a2","endpoints":["http://vm05.local:8001"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]}],"placement_targets":[{"name":"default-placement","tags":[],"storage_classes":["STANDARD"]}],"default_placement":"default-placement","realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","sync_policy":{"groups":[]},"enabled_features":["notification_v2","resharding"]},{"id":"b","name":"b","api_name":"b","is_master":false,"endpoints":["http://vm07.local:8000"],"hostnames":[],"hostnames_s3website":[],"master_zone":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","zones":[{"id":"cd52b229-0d4a-43bf-b4ba-156aa19e7aef","name":"b2","endpoints":["http://vm08.local:8001"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]},{"id":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","name":"b1","endpoints":["http://vm07.local:8000"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]}],"placement_targets":[{"name":"default-placement","tags":[],"storage_classes":["STANDARD"]}],"default_placement":"default-placement","realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","sync_policy":{"groups":[]},"enabled_features":["notification_v2","resharding"]}],"short_zone_ids":[{"key":"4b634ebf-58f5-45bc-839a-cb4b425e773a","val":162505008},{"key":"51055576-6662-409d-8abb-ecaeefd3dd80","val":2776933003},{"key":"cd52b229-0d4a-43bf-b4ba-156aa19e7aef","val":2686426109},{"key":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","val":114662526}]},"master_zonegroup":"a","master_zone":"4b634ebf-58f5-45bc-839a-cb4b425e773a","period_config":{"bucket_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false},"bucket_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false},"anonymous_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false}},"realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","realm_epoch":2} 2026-03-20T11:39:53.110 INFO:rgw_multi.tests:Set redirect_zone zone a2 to "" 2026-03-20T11:41:23.114 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_multi_zone_redirect ... ok 2026-03-20T11:41:23.248 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:41:23.248 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:41:23.248 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:41:23.285 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:41:23.285 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:41:23.388 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:41:23.399 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774006611.864443_171.1","last_update":"2026-03-20T11:36:51.864443Z"},{"marker":"1_1774006611.861570_128.1","last_update":"2026-03-20T11:36:51.861570Z"}] 2026-03-20T11:41:23.399 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:41:23.399 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:41:23.399 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:41:23.399 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:41:23.477 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:41:23.478 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:41:23.563 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:41:23.565+0000 7f27a2d2d900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:41:23.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:41:23.566+0000 7f27a2d2d900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:41:23.575 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006611.864443_171.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.864443Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006611.861570_128.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.861570Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:41:23Z"} 2026-03-20T11:41:23.575 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:41:23.575 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zone', 'create', '--rgw-zone', 'remove', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--access-key', 'GRL33KL1ULN5795C', '--secret', 'FPGvuVSJOG1341zbJPfw7ch710f3izIf', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:41:23.575 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zone', 'create', '--rgw-zone', 'remove', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--access-key', 'GRL33KL1ULN5795C', '--secret', 'FPGvuVSJOG1341zbJPfw7ch710f3izIf', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:41:23.576 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zone create --rgw-zone remove --rgw-zonegroup a --rgw-realm test-realm --access-key GRL33KL1ULN5795C --secret FPGvuVSJOG1341zbJPfw7ch710f3izIf --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:41:23.657 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:41:23.657 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:41:23.672 INFO:teuthology.orchestra.run.vm02.stderr:NOTICE: set zone's realm_id=6a789b89-c2b8-45f5-82cc-fb80234d0c6c 2026-03-20T11:41:23.726 INFO:teuthology.orchestra.run.vm02.stdout:{"id":"2ac988c3-b299-497d-8bc6-dcbd7d3a8d0c","name":"remove","domain_root":"remove.rgw.meta:root","control_pool":"remove.rgw.control","dedup_pool":"remove.rgw.dedup","gc_pool":"remove.rgw.log:gc","lc_pool":"remove.rgw.log:lc","log_pool":"remove.rgw.log","intent_log_pool":"remove.rgw.log:intent","usage_log_pool":"remove.rgw.log:usage","roles_pool":"remove.rgw.meta:roles","reshard_pool":"remove.rgw.log:reshard","user_keys_pool":"remove.rgw.meta:users.keys","user_email_pool":"remove.rgw.meta:users.email","user_swift_pool":"remove.rgw.meta:users.swift","user_uid_pool":"remove.rgw.meta:users.uid","otp_pool":"remove.rgw.otp","notif_pool":"remove.rgw.log:notif","topics_pool":"remove.rgw.meta:topics","account_pool":"remove.rgw.meta:accounts","group_pool":"remove.rgw.meta:groups","system_key":{"access_key":"GRL33KL1ULN5795C","secret_key":"FPGvuVSJOG1341zbJPfw7ch710f3izIf"},"placement_pools":[{"key":"default-placement","val":{"index_pool":"remove.rgw.buckets.index","storage_classes":{"STANDARD":{"data_pool":"remove.rgw.buckets.data"}},"data_extra_pool":"remove.rgw.buckets.non-ec","index_type":0,"inline_data":true}}],"realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","restore_pool":"remove.rgw.log:restore"} 2026-03-20T11:41:23.726 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['period', 'update', '--rgw-zone', 'remove', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:41:23.726 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'period', 'update', '--rgw-zone', 'remove', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:41:23.726 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 period update --rgw-zone remove --rgw-zonegroup a --rgw-realm test-realm --commit --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:41:23.805 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:41:23.805 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:41:23.824 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:41:23.826+0000 7fb618f34900 0 period (cb2fcdce-2c1b-429a-8601-a284ee3a2381 does not have zone 2ac988c3-b299-497d-8bc6-dcbd7d3a8d0c configured 2026-03-20T11:41:30.501 INFO:teuthology.orchestra.run.vm02.stderr:Sending period to new master zone 4b634ebf-58f5-45bc-839a-cb4b425e773a 2026-03-20T11:41:30.543 INFO:teuthology.orchestra.run.vm02.stdout:{"id":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","epoch":9,"predecessor_uuid":"5d4b2b15-cd70-4a63-87c8-e59369f9ab35","sync_status":[],"period_map":{"id":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","zonegroups":[{"id":"a","name":"a","api_name":"a","is_master":true,"endpoints":["http://vm02.local:8000"],"hostnames":[],"hostnames_s3website":[],"master_zone":"4b634ebf-58f5-45bc-839a-cb4b425e773a","zones":[{"id":"2ac988c3-b299-497d-8bc6-dcbd7d3a8d0c","name":"remove","endpoints":[],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]},{"id":"4b634ebf-58f5-45bc-839a-cb4b425e773a","name":"a1","endpoints":["http://vm02.local:8000"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]},{"id":"51055576-6662-409d-8abb-ecaeefd3dd80","name":"a2","endpoints":["http://vm05.local:8001"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]}],"placement_targets":[{"name":"default-placement","tags":[],"storage_classes":["STANDARD"]}],"default_placement":"default-placement","realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","sync_policy":{"groups":[]},"enabled_features":["notification_v2","resharding"]},{"id":"b","name":"b","api_name":"b","is_master":false,"endpoints":["http://vm07.local:8000"],"hostnames":[],"hostnames_s3website":[],"master_zone":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","zones":[{"id":"cd52b229-0d4a-43bf-b4ba-156aa19e7aef","name":"b2","endpoints":["http://vm08.local:8001"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]},{"id":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","name":"b1","endpoints":["http://vm07.local:8000"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]}],"placement_targets":[{"name":"default-placement","tags":[],"storage_classes":["STANDARD"]}],"default_placement":"default-placement","realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","sync_policy":{"groups":[]},"enabled_features":["notification_v2","resharding"]}],"short_zone_ids":[{"key":"2ac988c3-b299-497d-8bc6-dcbd7d3a8d0c","val":930242014},{"key":"4b634ebf-58f5-45bc-839a-cb4b425e773a","val":162505008},{"key":"51055576-6662-409d-8abb-ecaeefd3dd80","val":2776933003},{"key":"cd52b229-0d4a-43bf-b4ba-156aa19e7aef","val":2686426109},{"key":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","val":114662526}]},"master_zonegroup":"a","master_zone":"4b634ebf-58f5-45bc-839a-cb4b425e773a","period_config":{"bucket_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false},"bucket_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false},"anonymous_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false}},"realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","realm_epoch":2} 2026-03-20T11:41:30.543 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zonegroup', 'remove', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--rgw-zone', 'remove', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:41:30.543 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zonegroup', 'remove', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--rgw-zone', 'remove', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:41:30.543 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zonegroup remove --rgw-zonegroup a --rgw-realm test-realm --rgw-zone remove --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:41:30.623 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:41:30.623 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:41:30.647 INFO:teuthology.orchestra.run.vm02.stdout:{"id":"a","name":"a","api_name":"a","is_master":true,"endpoints":["http://vm02.local:8000"],"hostnames":[],"hostnames_s3website":[],"master_zone":"4b634ebf-58f5-45bc-839a-cb4b425e773a","zones":[{"id":"4b634ebf-58f5-45bc-839a-cb4b425e773a","name":"a1","endpoints":["http://vm02.local:8000"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]},{"id":"51055576-6662-409d-8abb-ecaeefd3dd80","name":"a2","endpoints":["http://vm05.local:8001"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]}],"placement_targets":[{"name":"default-placement","tags":[],"storage_classes":["STANDARD"]}],"default_placement":"default-placement","realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","sync_policy":{"groups":[]},"enabled_features":["notification_v2","resharding"]} 2026-03-20T11:41:30.647 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zonegroup', 'remove', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--rgw-zone', 'remove', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:41:30.647 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zonegroup', 'remove', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--rgw-zone', 'remove', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:41:30.647 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zonegroup remove --rgw-zonegroup a --rgw-realm test-realm --rgw-zone remove --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:41:30.726 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:41:30.726 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:41:30.740 INFO:teuthology.orchestra.run.vm02.stderr:zone name remove not found in zonegroup a 2026-03-20T11:41:30.746 DEBUG:teuthology.orchestra.run:got remote process result: 2 2026-03-20T11:41:30.750 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zone', 'delete', '--rgw-zone', 'remove', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:41:30.750 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zone', 'delete', '--rgw-zone', 'remove', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:41:30.750 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zone delete --rgw-zone remove --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:41:30.826 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:41:30.826 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:41:30.863 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['period', 'update', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:41:30.863 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'period', 'update', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:41:30.863 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 period update --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --commit --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:41:30.942 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:41:30.942 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:41:30.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:41:30.964+0000 7fa60de78900 0 WARNING: can't generate connection for zone 2ac988c3-b299-497d-8bc6-dcbd7d3a8d0c id remove: no endpoints defined 2026-03-20T11:41:31.065 INFO:teuthology.orchestra.run.vm02.stdout:{"id":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","epoch":10,"predecessor_uuid":"5d4b2b15-cd70-4a63-87c8-e59369f9ab35","sync_status":[],"period_map":{"id":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","zonegroups":[{"id":"a","name":"a","api_name":"a","is_master":true,"endpoints":["http://vm02.local:8000"],"hostnames":[],"hostnames_s3website":[],"master_zone":"4b634ebf-58f5-45bc-839a-cb4b425e773a","zones":[{"id":"4b634ebf-58f5-45bc-839a-cb4b425e773a","name":"a1","endpoints":["http://vm02.local:8000"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]},{"id":"51055576-6662-409d-8abb-ecaeefd3dd80","name":"a2","endpoints":["http://vm05.local:8001"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]}],"placement_targets":[{"name":"default-placement","tags":[],"storage_classes":["STANDARD"]}],"default_placement":"default-placement","realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","sync_policy":{"groups":[]},"enabled_features":["notification_v2","resharding"]},{"id":"b","name":"b","api_name":"b","is_master":false,"endpoints":["http://vm07.local:8000"],"hostnames":[],"hostnames_s3website":[],"master_zone":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","zones":[{"id":"cd52b229-0d4a-43bf-b4ba-156aa19e7aef","name":"b2","endpoints":["http://vm08.local:8001"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]},{"id":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","name":"b1","endpoints":["http://vm07.local:8000"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]}],"placement_targets":[{"name":"default-placement","tags":[],"storage_classes":["STANDARD"]}],"default_placement":"default-placement","realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","sync_policy":{"groups":[]},"enabled_features":["notification_v2","resharding"]}],"short_zone_ids":[{"key":"4b634ebf-58f5-45bc-839a-cb4b425e773a","val":162505008},{"key":"51055576-6662-409d-8abb-ecaeefd3dd80","val":2776933003},{"key":"cd52b229-0d4a-43bf-b4ba-156aa19e7aef","val":2686426109},{"key":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","val":114662526}]},"master_zonegroup":"a","master_zone":"4b634ebf-58f5-45bc-839a-cb4b425e773a","period_config":{"bucket_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false},"bucket_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false},"anonymous_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false}},"realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","realm_epoch":2} 2026-03-20T11:41:31.068 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_zonegroup_remove ... ok 2026-03-20T11:41:31.068 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zonegroup', 'create', '--rgw-zonegroup', 'remove_zg', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:41:31.068 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zonegroup', 'create', '--rgw-zonegroup', 'remove_zg', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:41:31.068 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zonegroup create --rgw-zonegroup remove_zg --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:41:31.142 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:41:31.143 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:41:31.166 INFO:teuthology.orchestra.run.vm02.stdout:{"id":"4658d461-f8d7-4e67-ba4f-3ffd04e2157a","name":"remove_zg","api_name":"remove_zg","is_master":false,"endpoints":[],"hostnames":[],"hostnames_s3website":[],"master_zone":"","zones":[],"placement_targets":[{"name":"default-placement","tags":[],"storage_classes":[]}],"default_placement":"default-placement","realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","sync_policy":{"groups":[]},"enabled_features":["notification_v2","resharding"]} 2026-03-20T11:41:31.167 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zone', 'create', '--rgw-zone', 'remove', '--rgw-zonegroup', 'remove_zg', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:41:31.167 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zone', 'create', '--rgw-zone', 'remove', '--rgw-zonegroup', 'remove_zg', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:41:31.167 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zone create --rgw-zone remove --rgw-zonegroup remove_zg --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:41:31.244 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:41:31.244 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:41:31.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:41:31.275+0000 7f8749422900 0 NOTICE: promoted remove as new master_zone of zonegroup remove_zg 2026-03-20T11:41:31.280 INFO:teuthology.orchestra.run.vm02.stdout:{"id":"af868862-cdaf-40b7-826a-5e14f4813cf7","name":"remove","domain_root":"remove.rgw.meta:root","control_pool":"remove.rgw.control","dedup_pool":"remove.rgw.dedup","gc_pool":"remove.rgw.log:gc","lc_pool":"remove.rgw.log:lc","log_pool":"remove.rgw.log","intent_log_pool":"remove.rgw.log:intent","usage_log_pool":"remove.rgw.log:usage","roles_pool":"remove.rgw.meta:roles","reshard_pool":"remove.rgw.log:reshard","user_keys_pool":"remove.rgw.meta:users.keys","user_email_pool":"remove.rgw.meta:users.email","user_swift_pool":"remove.rgw.meta:users.swift","user_uid_pool":"remove.rgw.meta:users.uid","otp_pool":"remove.rgw.otp","notif_pool":"remove.rgw.log:notif","topics_pool":"remove.rgw.meta:topics","account_pool":"remove.rgw.meta:accounts","group_pool":"remove.rgw.meta:groups","system_key":{"access_key":"","secret_key":""},"placement_pools":[{"key":"default-placement","val":{"index_pool":"remove.rgw.buckets.index","storage_classes":{"STANDARD":{"data_pool":"remove.rgw.buckets.data"}},"data_extra_pool":"remove.rgw.buckets.non-ec","index_type":0,"inline_data":true}}],"realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","restore_pool":"remove.rgw.log:restore"} 2026-03-20T11:41:31.280 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['period', 'update', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:41:31.280 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'period', 'update', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:41:31.281 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 period update --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --commit --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:41:31.359 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:41:31.359 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:41:31.627 INFO:teuthology.orchestra.run.vm02.stdout:{"id":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","epoch":11,"predecessor_uuid":"5d4b2b15-cd70-4a63-87c8-e59369f9ab35","sync_status":[],"period_map":{"id":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","zonegroups":[{"id":"4658d461-f8d7-4e67-ba4f-3ffd04e2157a","name":"remove_zg","api_name":"remove_zg","is_master":false,"endpoints":[],"hostnames":[],"hostnames_s3website":[],"master_zone":"af868862-cdaf-40b7-826a-5e14f4813cf7","zones":[{"id":"af868862-cdaf-40b7-826a-5e14f4813cf7","name":"remove","endpoints":[],"log_meta":false,"log_data":false,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]}],"placement_targets":[{"name":"default-placement","tags":[],"storage_classes":["STANDARD"]}],"default_placement":"default-placement","realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","sync_policy":{"groups":[]},"enabled_features":["notification_v2","resharding"]},{"id":"a","name":"a","api_name":"a","is_master":true,"endpoints":["http://vm02.local:8000"],"hostnames":[],"hostnames_s3website":[],"master_zone":"4b634ebf-58f5-45bc-839a-cb4b425e773a","zones":[{"id":"4b634ebf-58f5-45bc-839a-cb4b425e773a","name":"a1","endpoints":["http://vm02.local:8000"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]},{"id":"51055576-6662-409d-8abb-ecaeefd3dd80","name":"a2","endpoints":["http://vm05.local:8001"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]}],"placement_targets":[{"name":"default-placement","tags":[],"storage_classes":["STANDARD"]}],"default_placement":"default-placement","realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","sync_policy":{"groups":[]},"enabled_features":["notification_v2","resharding"]},{"id":"b","name":"b","api_name":"b","is_master":false,"endpoints":["http://vm07.local:8000"],"hostnames":[],"hostnames_s3website":[],"master_zone":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","zones":[{"id":"cd52b229-0d4a-43bf-b4ba-156aa19e7aef","name":"b2","endpoints":["http://vm08.local:8001"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]},{"id":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","name":"b1","endpoints":["http://vm07.local:8000"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]}],"placement_targets":[{"name":"default-placement","tags":[],"storage_classes":["STANDARD"]}],"default_placement":"default-placement","realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","sync_policy":{"groups":[]},"enabled_features":["notification_v2","resharding"]}],"short_zone_ids":[{"key":"4b634ebf-58f5-45bc-839a-cb4b425e773a","val":162505008},{"key":"51055576-6662-409d-8abb-ecaeefd3dd80","val":2776933003},{"key":"af868862-cdaf-40b7-826a-5e14f4813cf7","val":1983292174},{"key":"cd52b229-0d4a-43bf-b4ba-156aa19e7aef","val":2686426109},{"key":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","val":114662526}]},"master_zonegroup":"a","master_zone":"4b634ebf-58f5-45bc-839a-cb4b425e773a","period_config":{"bucket_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false},"bucket_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false},"anonymous_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false}},"realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","realm_epoch":2} 2026-03-20T11:41:31.628 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zone', 'delete', '--rgw-zone', 'remove', '--rgw-zonegroup', 'remove_zg', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:41:31.628 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zone', 'delete', '--rgw-zone', 'remove', '--rgw-zonegroup', 'remove_zg', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:41:31.628 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zone delete --rgw-zone remove --rgw-zonegroup remove_zg --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:41:31.706 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:41:31.731 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:41:31.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:41:31.730+0000 7f243e9d0900 0 NOTICE: removed master_zone of zonegroup remove_zg 2026-03-20T11:41:31.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:41:31.740+0000 7f243e9d0900 0 Removed zone from zonegroup remove_zg 2026-03-20T11:41:31.802 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['period', 'update', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:41:31.802 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'period', 'update', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:41:31.802 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 period update --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:41:31.877 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:41:31.877 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:41:31.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:41:31.900+0000 7ff945728900 0 ERROR: zonegroup remove_zg has a non existent master zone 2026-03-20T11:41:31.902 DEBUG:teuthology.orchestra.run:got remote process result: 22 2026-03-20T11:41:31.902 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zonegroup', 'delete', '--rgw-zonegroup', 'remove_zg', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:41:31.902 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zonegroup', 'delete', '--rgw-zonegroup', 'remove_zg', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:41:31.902 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zonegroup delete --rgw-zonegroup remove_zg --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:41:31.980 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:41:31.980 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:41:32.016 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['period', 'update', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:41:32.016 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'period', 'update', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:41:32.016 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 period update --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --commit --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:41:32.090 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:41:32.091 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:41:32.347 INFO:teuthology.orchestra.run.vm02.stdout:{"id":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","epoch":12,"predecessor_uuid":"5d4b2b15-cd70-4a63-87c8-e59369f9ab35","sync_status":[],"period_map":{"id":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","zonegroups":[{"id":"a","name":"a","api_name":"a","is_master":true,"endpoints":["http://vm02.local:8000"],"hostnames":[],"hostnames_s3website":[],"master_zone":"4b634ebf-58f5-45bc-839a-cb4b425e773a","zones":[{"id":"4b634ebf-58f5-45bc-839a-cb4b425e773a","name":"a1","endpoints":["http://vm02.local:8000"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]},{"id":"51055576-6662-409d-8abb-ecaeefd3dd80","name":"a2","endpoints":["http://vm05.local:8001"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]}],"placement_targets":[{"name":"default-placement","tags":[],"storage_classes":["STANDARD"]}],"default_placement":"default-placement","realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","sync_policy":{"groups":[]},"enabled_features":["notification_v2","resharding"]},{"id":"b","name":"b","api_name":"b","is_master":false,"endpoints":["http://vm07.local:8000"],"hostnames":[],"hostnames_s3website":[],"master_zone":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","zones":[{"id":"cd52b229-0d4a-43bf-b4ba-156aa19e7aef","name":"b2","endpoints":["http://vm08.local:8001"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]},{"id":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","name":"b1","endpoints":["http://vm07.local:8000"],"log_meta":false,"log_data":true,"bucket_index_max_shards":11,"read_only":false,"tier_type":"","sync_from_all":true,"sync_from":[],"redirect_zone":"","supported_features":["compress-encrypted","notification_v2","resharding"]}],"placement_targets":[{"name":"default-placement","tags":[],"storage_classes":["STANDARD"]}],"default_placement":"default-placement","realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","sync_policy":{"groups":[]},"enabled_features":["notification_v2","resharding"]}],"short_zone_ids":[{"key":"4b634ebf-58f5-45bc-839a-cb4b425e773a","val":162505008},{"key":"51055576-6662-409d-8abb-ecaeefd3dd80","val":2776933003},{"key":"cd52b229-0d4a-43bf-b4ba-156aa19e7aef","val":2686426109},{"key":"efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c","val":114662526}]},"master_zonegroup":"a","master_zone":"4b634ebf-58f5-45bc-839a-cb4b425e773a","period_config":{"bucket_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1},"user_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false},"bucket_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false},"anonymous_ratelimit":{"max_read_ops":0,"max_write_ops":0,"max_read_bytes":0,"max_write_bytes":0,"enabled":false}},"realm_id":"6a789b89-c2b8-45f5-82cc-fb80234d0c6c","realm_epoch":2} 2026-03-20T11:41:32.347 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_zg_master_zone_delete ... ok 2026-03-20T11:41:32.477 INFO:rgw_multi.tests:create bucket zone=a1 name=dipepu-41 2026-03-20T11:41:46.025 INFO:rgw_multi.tests:create bucket zone=a2 name=dipepu-42 2026-03-20T11:41:46.161 INFO:rgw_multi.tests:create bucket zone=b1 name=dipepu-43 2026-03-20T11:42:16.421 INFO:rgw_multi.tests:create bucket zone=b2 name=dipepu-44 2026-03-20T11:42:16.452 INFO:rgw_multi.tests:meta checkpoint 2026-03-20T11:42:16.452 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:42:16.452 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:42:16.452 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:42:16.492 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:42:16.492 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:42:16.588 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:42:16.600 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774006936.439637_219.1","last_update":"2026-03-20T11:42:16.439637Z"},{"marker":"1_1774006936.435420_200.1","last_update":"2026-03-20T11:42:16.435420Z"}] 2026-03-20T11:42:16.600 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:42:16.600 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:42:16.600 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:42:16.600 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:42:16.679 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:42:16.679 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:42:16.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:42:16.766+0000 7f88f853c900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:42:16.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:42:16.767+0000 7f88f853c900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:42:16.779 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006936.439637_219.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:42:16.439637Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006936.435420_200.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:42:16.435420Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:42:16Z"} 2026-03-20T11:42:16.779 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:42:16.779 INFO:rgw_multi.tests:starting meta checkpoint for zone=b1 2026-03-20T11:42:16.779 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:42:16.779 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:42:16.779 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:42:16.820 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:42:16.820 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:42:16.891 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:42:16.894+0000 7f800492e900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:42:16.892 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:42:16.895+0000 7f800492e900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:42:16.902 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006611.864443_171.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.864443Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006611.861570_128.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.861570Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:42:16Z"} 2026-03-20T11:42:16.902 WARNING:rgw_multi.tests:zone b1 behind master: shard=2 master=1_1774006936.439637_219.1 target=1_1774006611.864443_171.1, shard=3 master=1_1774006936.435420_200.1 target=1_1774006611.861570_128.1 2026-03-20T11:42:21.903 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:42:21.903 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:42:21.903 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:42:21.944 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:42:21.944 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:42:22.011 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:42:22.014+0000 7f81c4141900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:42:22.012 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:42:22.015+0000 7f81c4141900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:42:22.022 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006611.864443_171.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.864443Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006611.861570_128.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.861570Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:42:22Z"} 2026-03-20T11:42:22.022 WARNING:rgw_multi.tests:zone b1 behind master: shard=2 master=1_1774006936.439637_219.1 target=1_1774006611.864443_171.1, shard=3 master=1_1774006936.435420_200.1 target=1_1774006611.861570_128.1 2026-03-20T11:42:27.023 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:42:27.023 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:42:27.023 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:42:27.060 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:42:27.060 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:42:27.126 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:42:27.129+0000 7ff3d0f28900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:42:27.127 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:42:27.130+0000 7ff3d0f28900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:42:27.135 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006611.864443_171.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.864443Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006611.861570_128.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.861570Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:42:27Z"} 2026-03-20T11:42:27.136 WARNING:rgw_multi.tests:zone b1 behind master: shard=2 master=1_1774006936.439637_219.1 target=1_1774006611.864443_171.1, shard=3 master=1_1774006936.435420_200.1 target=1_1774006611.861570_128.1 2026-03-20T11:42:32.136 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:42:32.136 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:42:32.136 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:42:32.174 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:42:32.174 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:42:32.239 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:42:32.242+0000 7f2141a20900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:42:32.240 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:42:32.243+0000 7f2141a20900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:42:32.250 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006611.864443_171.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.864443Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006611.861570_128.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.861570Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:42:32Z"} 2026-03-20T11:42:32.250 WARNING:rgw_multi.tests:zone b1 behind master: shard=2 master=1_1774006936.439637_219.1 target=1_1774006611.864443_171.1, shard=3 master=1_1774006936.435420_200.1 target=1_1774006611.861570_128.1 2026-03-20T11:42:37.251 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:42:37.251 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:42:37.251 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:42:37.290 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:42:37.290 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:42:37.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:42:37.367+0000 7f3d64e10900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:42:37.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:42:37.368+0000 7f3d64e10900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:42:37.376 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006611.864443_171.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.864443Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006611.861570_128.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.861570Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:42:37Z"} 2026-03-20T11:42:37.376 WARNING:rgw_multi.tests:zone b1 behind master: shard=2 master=1_1774006936.439637_219.1 target=1_1774006611.864443_171.1, shard=3 master=1_1774006936.435420_200.1 target=1_1774006611.861570_128.1 2026-03-20T11:42:42.376 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:42:42.377 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:42:42.377 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:42:42.416 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:42:42.416 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:42:42.489 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:42:42.492+0000 7f65b2933900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:42:42.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:42:42.493+0000 7f65b2933900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:42:42.501 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006611.864443_171.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.864443Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006611.861570_128.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.861570Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:42:42Z"} 2026-03-20T11:42:42.502 WARNING:rgw_multi.tests:zone b1 behind master: shard=2 master=1_1774006936.439637_219.1 target=1_1774006611.864443_171.1, shard=3 master=1_1774006936.435420_200.1 target=1_1774006611.861570_128.1 2026-03-20T11:42:47.502 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:42:47.502 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:42:47.502 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:42:47.545 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:42:47.545 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:42:47.620 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:42:47.623+0000 7fc9be93b900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:42:47.621 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:42:47.624+0000 7fc9be93b900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:42:47.632 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006611.864443_171.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.864443Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006611.861570_128.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.861570Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:42:47Z"} 2026-03-20T11:42:47.632 WARNING:rgw_multi.tests:zone b1 behind master: shard=2 master=1_1774006936.439637_219.1 target=1_1774006611.864443_171.1, shard=3 master=1_1774006936.435420_200.1 target=1_1774006611.861570_128.1 2026-03-20T11:42:52.632 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:42:52.632 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:42:52.633 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:42:52.672 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:42:52.672 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:42:52.747 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:42:52.750+0000 7fe006a10900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:42:52.748 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:42:52.750+0000 7fe006a10900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:42:52.757 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006611.864443_171.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.864443Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006611.861570_128.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.861570Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:42:52Z"} 2026-03-20T11:42:52.757 WARNING:rgw_multi.tests:zone b1 behind master: shard=2 master=1_1774006936.439637_219.1 target=1_1774006611.864443_171.1, shard=3 master=1_1774006936.435420_200.1 target=1_1774006611.861570_128.1 2026-03-20T11:42:57.758 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:42:57.758 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:42:57.759 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:42:57.800 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:42:57.800 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:42:57.878 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:42:57.881+0000 7f8428612900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:42:57.879 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:42:57.882+0000 7f8428612900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:42:57.889 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006611.864443_171.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.864443Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006611.861570_128.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.861570Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:42:57Z"} 2026-03-20T11:42:57.889 WARNING:rgw_multi.tests:zone b1 behind master: shard=2 master=1_1774006936.439637_219.1 target=1_1774006611.864443_171.1, shard=3 master=1_1774006936.435420_200.1 target=1_1774006611.861570_128.1 2026-03-20T11:43:02.890 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:02.895 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:02.895 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:43:02.935 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:43:02.935 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:43:03.026 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:43:03.029+0000 7f76d6130900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:03.027 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:43:03.030+0000 7f76d6130900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:03.038 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006611.864443_171.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.864443Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006611.861570_128.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.861570Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:43:03Z"} 2026-03-20T11:43:03.038 WARNING:rgw_multi.tests:zone b1 behind master: shard=2 master=1_1774006936.439637_219.1 target=1_1774006611.864443_171.1, shard=3 master=1_1774006936.435420_200.1 target=1_1774006611.861570_128.1 2026-03-20T11:43:08.038 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:08.038 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:08.038 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:43:08.082 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:43:08.082 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:43:08.168 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:43:08.170+0000 7fd0a2c10900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:08.168 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:43:08.171+0000 7fd0a2c10900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:08.179 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006611.864443_171.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.864443Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006611.861570_128.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.861570Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:43:08Z"} 2026-03-20T11:43:08.179 WARNING:rgw_multi.tests:zone b1 behind master: shard=2 master=1_1774006936.439637_219.1 target=1_1774006611.864443_171.1, shard=3 master=1_1774006936.435420_200.1 target=1_1774006611.861570_128.1 2026-03-20T11:43:13.180 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:13.180 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:13.180 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:43:13.220 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:43:13.220 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:43:13.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:43:13.317+0000 7fd8d1410900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:13.316 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:43:13.319+0000 7fd8d1410900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:13.326 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006611.864443_171.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.864443Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006611.861570_128.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.861570Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:43:13Z"} 2026-03-20T11:43:13.326 WARNING:rgw_multi.tests:zone b1 behind master: shard=2 master=1_1774006936.439637_219.1 target=1_1774006611.864443_171.1, shard=3 master=1_1774006936.435420_200.1 target=1_1774006611.861570_128.1 2026-03-20T11:43:18.327 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:18.327 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:18.327 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:43:18.368 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:43:18.368 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:43:18.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:43:18.452+0000 7fdd7c814900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:18.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:43:18.453+0000 7fdd7c814900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:18.462 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006611.864443_171.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.864443Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006611.861570_128.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.861570Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:43:18Z"} 2026-03-20T11:43:18.462 WARNING:rgw_multi.tests:zone b1 behind master: shard=2 master=1_1774006936.439637_219.1 target=1_1774006611.864443_171.1, shard=3 master=1_1774006936.435420_200.1 target=1_1774006611.861570_128.1 2026-03-20T11:43:23.463 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:23.463 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:23.463 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:43:23.507 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:43:23.507 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:43:23.592 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:43:23.595+0000 7f9ac8810900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:23.593 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:43:23.596+0000 7f9ac8810900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:23.605 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006611.864443_171.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.864443Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006611.861570_128.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.861570Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:43:23Z"} 2026-03-20T11:43:23.606 WARNING:rgw_multi.tests:zone b1 behind master: shard=2 master=1_1774006936.439637_219.1 target=1_1774006611.864443_171.1, shard=3 master=1_1774006936.435420_200.1 target=1_1774006611.861570_128.1 2026-03-20T11:43:28.607 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:28.607 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:28.607 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:43:28.652 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:43:28.652 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:43:28.723 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:43:28.726+0000 7f4740b26900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:28.724 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:43:28.727+0000 7f4740b26900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:28.735 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006611.864443_171.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.864443Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006611.861570_128.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.861570Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:43:28Z"} 2026-03-20T11:43:28.735 WARNING:rgw_multi.tests:zone b1 behind master: shard=2 master=1_1774006936.439637_219.1 target=1_1774006611.864443_171.1, shard=3 master=1_1774006936.435420_200.1 target=1_1774006611.861570_128.1 2026-03-20T11:43:33.735 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:33.735 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:33.736 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:43:33.777 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:43:33.777 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:43:33.882 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:43:33.885+0000 7fb8be810900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:33.883 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:43:33.886+0000 7fb8be810900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:33.893 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006611.864443_171.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.864443Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006611.861570_128.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.861570Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:43:33Z"} 2026-03-20T11:43:33.893 WARNING:rgw_multi.tests:zone b1 behind master: shard=2 master=1_1774006936.439637_219.1 target=1_1774006611.864443_171.1, shard=3 master=1_1774006936.435420_200.1 target=1_1774006611.861570_128.1 2026-03-20T11:43:38.893 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:38.894 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:38.894 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:43:38.933 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:43:38.933 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:43:39.005 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:43:39.008+0000 7fbc22b2a900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:39.006 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:43:39.009+0000 7fbc22b2a900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:39.017 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006611.864443_171.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.864443Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006611.861570_128.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.861570Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:43:39Z"} 2026-03-20T11:43:39.018 WARNING:rgw_multi.tests:zone b1 behind master: shard=2 master=1_1774006936.439637_219.1 target=1_1774006611.864443_171.1, shard=3 master=1_1774006936.435420_200.1 target=1_1774006611.861570_128.1 2026-03-20T11:43:44.018 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:44.019 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:44.019 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:43:44.061 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:43:44.061 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:43:44.143 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:43:44.146+0000 7f4cd5762900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:44.144 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:43:44.147+0000 7f4cd5762900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:44.157 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006611.864443_171.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.864443Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006611.861570_128.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.861570Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:43:44Z"} 2026-03-20T11:43:44.157 WARNING:rgw_multi.tests:zone b1 behind master: shard=2 master=1_1774006936.439637_219.1 target=1_1774006611.864443_171.1, shard=3 master=1_1774006936.435420_200.1 target=1_1774006611.861570_128.1 2026-03-20T11:43:49.157 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:49.157 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:49.157 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:43:49.195 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:43:49.196 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:43:49.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:43:49.268+0000 7fa70a521900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:49.266 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:43:49.269+0000 7fa70a521900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:49.275 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006936.439637_219.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:42:16.439637Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006936.435420_200.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:42:16.435420Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:43:49Z"} 2026-03-20T11:43:49.275 INFO:rgw_multi.tests:finish meta checkpoint for zone=b1 2026-03-20T11:43:49.275 INFO:rgw_multi.tests:starting meta checkpoint for zone=b2 2026-03-20T11:43:49.275 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:49.275 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:49.275 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b2 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:43:49.356 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:43:49.356 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:43:49.432 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:43:49.435+0000 7f8aa113d900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:49.433 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:43:49.436+0000 7f8aa113d900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:49.443 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006936.439637_219.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:42:16.439637Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006936.435420_200.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:42:16.435420Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:43:49Z"} 2026-03-20T11:43:49.444 INFO:rgw_multi.tests:finish meta checkpoint for zone=b2 2026-03-20T11:43:49.450 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_set_bucket_website ... SKIP: test_set_bucket_website skipped. Requires rgw_enable_static_website = 1. 2026-03-20T11:43:49.579 INFO:rgw_multi.tests:create bucket zone=a1 name=dipepu-45 2026-03-20T11:43:49.594 INFO:rgw_multi.tests:create bucket zone=a2 name=dipepu-46 2026-03-20T11:43:49.766 INFO:rgw_multi.tests:create bucket zone=b1 name=dipepu-47 2026-03-20T11:43:49.792 INFO:rgw_multi.tests:create bucket zone=b2 name=dipepu-48 2026-03-20T11:43:49.817 INFO:rgw_multi.tests:meta checkpoint 2026-03-20T11:43:49.818 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:49.818 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:49.818 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:43:49.863 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:43:49.864 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:43:49.953 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:43:49.964 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774007029.809335_223.1","last_update":"2026-03-20T11:43:49.809335Z"},{"marker":"1_1774007029.806454_204.1","last_update":"2026-03-20T11:43:49.806454Z"}] 2026-03-20T11:43:49.964 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:43:49.964 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:49.964 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:49.964 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:43:50.007 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:43:50.007 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:43:50.095 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:43:50.098+0000 7fee85f26900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:50.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:43:50.098+0000 7fee85f26900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:50.107 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774007029.809335_223.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:49.809335Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774007029.806454_204.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:49.806454Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:43:50Z"} 2026-03-20T11:43:50.107 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:43:50.107 INFO:rgw_multi.tests:starting meta checkpoint for zone=b1 2026-03-20T11:43:50.107 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:50.107 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:50.107 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:43:50.148 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:43:50.149 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:43:50.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:43:50.226+0000 7f5e60525900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:50.225 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:43:50.228+0000 7f5e60525900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:50.236 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774006936.439637_219.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:42:16.439637Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774006936.435420_200.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:42:16.435420Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:43:50Z"} 2026-03-20T11:43:50.236 WARNING:rgw_multi.tests:zone b1 behind master: shard=2 master=1_1774007029.809335_223.1 target=1_1774006936.439637_219.1, shard=3 master=1_1774007029.806454_204.1 target=1_1774006936.435420_200.1 2026-03-20T11:43:55.237 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:55.238 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:55.238 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:43:55.278 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:43:55.278 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:43:55.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:43:55.359+0000 7eff61410900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:55.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:43:55.359+0000 7eff61410900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:55.368 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774007029.809335_223.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:49.809335Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774007029.806454_204.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:49.806454Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:43:55Z"} 2026-03-20T11:43:55.369 INFO:rgw_multi.tests:finish meta checkpoint for zone=b1 2026-03-20T11:43:55.369 INFO:rgw_multi.tests:starting meta checkpoint for zone=b2 2026-03-20T11:43:55.369 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:55.369 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:55.369 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b2 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:43:55.454 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:43:55.455 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:43:55.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:43:55.530+0000 7facce556900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:55.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:43:55.530+0000 7facce556900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:55.542 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774007029.809335_223.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:49.809335Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774007029.806454_204.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:49.806454Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:43:55Z"} 2026-03-20T11:43:55.542 INFO:rgw_multi.tests:finish meta checkpoint for zone=b2 2026-03-20T11:43:58.765 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_set_bucket_policy ... ok 2026-03-20T11:43:58.819 INFO:rgw_multi.tests:create bucket zone=a1 name=dipepu-49 2026-03-20T11:43:58.834 INFO:rgw_multi.tests:create bucket zone=a2 name=dipepu-50 2026-03-20T11:43:58.861 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:58.861 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:58.861 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:43:58.898 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:43:58.898 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:43:58.972 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:43:58.984 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774007038.854251_226.1","last_update":"2026-03-20T11:43:58.854251Z"},{"marker":"1_1774007038.831325_209.1","last_update":"2026-03-20T11:43:58.831325Z"}] 2026-03-20T11:43:58.984 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:43:58.984 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:58.984 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:58.984 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:43:59.065 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:43:59.065 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:43:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:43:59.142+0000 7f4fda522900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:43:59.143+0000 7f4fda522900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:43:59.151 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774007038.854251_226.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:58.854251Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774007038.831325_209.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:58.831325Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:43:59Z"} 2026-03-20T11:43:59.151 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:43:59.151 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'disable', '--bucket', 'dipepu-49', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:43:59.151 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'disable', '--bucket', 'dipepu-49', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:43:59.151 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync disable --bucket dipepu-49 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:43:59.230 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:43:59.230 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:43:59.345 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'disable', '--bucket', 'dipepu-50', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:43:59.345 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'disable', '--bucket', 'dipepu-50', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:43:59.345 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync disable --bucket dipepu-50 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:43:59.425 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:43:59.425 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:43:59.541 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['log', 'list', '--rgw-zone', 'a1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:59.541 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'log', 'list', '--rgw-zone', 'a1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:59.541 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 log list --rgw-zone a1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:43:59.618 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:43:59.618 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:43:59.722 INFO:teuthology.orchestra.run.vm02.stdout:["obj_delete_at_hint.0000000078","meta.history","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19:8","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27:9","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24:5","obj_delete_at_hint.0000000070","obj_delete_at_hint.0000000104","data_log.2.4tpW0ZVU/LY.0","obj_delete_at_hint.0000000026","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27:4","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43:7","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19:10","datalog.sync-status.shard.51055576-6662-409d-8abb-ecaeefd3dd80.3","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22:7","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26:5","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24:9","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41:10","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23:9","obj_delete_at_hint.0000000028","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13:5","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17:3","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44:3","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44:10","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19:7","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24:6","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42:8","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14:3","obj_delete_at_hint.0000000040","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12:5","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22:1","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26:10","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16:0","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17:5","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43:1","obj_delete_at_hint.0000000015","obj_delete_at_hint.0000000069","obj_delete_at_hint.0000000095","obj_delete_at_hint.0000000003","obj_delete_at_hint.0000000047","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19:5","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41:6","obj_delete_at_hint.0000000035","obj_delete_at_hint.0000000037","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28:0","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41:1","obj_delete_at_hint.0000000024","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12:2","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17:7","obj_delete_at_hint.0000000000","obj_delete_at_hint.0000000031","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27:6","data_log.3","datalog.sync-status.shard.51055576-6662-409d-8abb-ecaeefd3dd80.1","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22:5","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26:3","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14:9","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16:5","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21:7","data_loggenerations_metadata","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22:0","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19:2","obj_delete_at_hint.0000000076","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28:8","obj_delete_at_hint.0000000113","obj_delete_at_hint.0000000005","data_log.3.X+xphe8wIWt.0","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18:0","obj_delete_at_hint.0000000011","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17:1","obj_delete_at_hint.0000000058","obj_delete_at_hint.0000000052","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41:9","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43:9","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12:6","obj_delete_at_hint.0000000088","bucket.full-sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17","obj_delete_at_hint.0000000080","obj_delete_at_hint.0000000090","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42:6","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42:1","obj_delete_at_hint.0000000110","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12:10","obj_delete_at_hint.0000000096","_sem_setdata_log.3","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28:9","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1:7","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22:9","obj_delete_at_hint.0000000087","obj_delete_at_hint.0000000008","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28:3","meta.log.cb2fcdce-2c1b-429a-8601-a284ee3a2381.0","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27:5","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25:2","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12:1","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21:2","obj_delete_at_hint.0000000006","obj_delete_at_hint.0000000029","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27:10","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1:2","obj_delete_at_hint.0000000089","obj_delete_at_hint.0000000022","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44:6","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18:6","datalog.sync-status.shard.51055576-6662-409d-8abb-ecaeefd3dd80.2.retry","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18:1","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14:6","bilog.trim","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13:4","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20:4","obj_delete_at_hint.0000000016","obj_delete_at_hint.0000000041","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44:8","bucket.full-sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44","obj_delete_at_hint.0000000018","obj_delete_at_hint.0000000092","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44:9","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21:0","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15:3","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43:8","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21:5","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25:3","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20:1","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42:9","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44:7","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1:3","obj_delete_at_hint.0000000014","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41:2","_sem_setdata_log.0","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25:7","obj_delete_at_hint.0000000112","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43:5","datalog.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80","obj_delete_at_hint.0000000007","obj_delete_at_hint.0000000021","datalog.sync-status.shard.51055576-6662-409d-8abb-ecaeefd3dd80.2","obj_delete_at_hint.0000000064","obj_delete_at_hint.0000000071","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13:8","obj_delete_at_hint.0000000074","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21:8","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43:0","data_log.1.eFJ6hNrdN2M.0","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15:9","obj_delete_at_hint.0000000081","obj_delete_at_hint.0000000009","obj_delete_at_hint.0000000121","data_log.1","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16:6","bucket.full-sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1:6","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1:4","obj_delete_at_hint.0000000125","obj_delete_at_hint.0000000082","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26:0","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21:3","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14:7","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25:5","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1:9","obj_delete_at_hint.0000000105","obj_delete_at_hint.0000000059","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19:1","datalog.sync-status.shard.51055576-6662-409d-8abb-ecaeefd3dd80.3.retry","obj_delete_at_hint.0000000077","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42:5","meta.log.cb2fcdce-2c1b-429a-8601-a284ee3a2381.1","obj_delete_at_hint.0000000032","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23:6","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27:2","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25:8","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20:7","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11:0:1","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15:4","obj_delete_at_hint.0000000053","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18:5","obj_delete_at_hint.0000000091","obj_delete_at_hint.0000000065","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17:6","obj_delete_at_hint.0000000083","obj_delete_at_hint.0000000010","bucket.full-sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28:6","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21:4","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15:10","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27:7","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28:10","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43:6","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18:8","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22:3","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17:4","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22:2","obj_delete_at_hint.0000000045","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22:8","obj_delete_at_hint.0000000002","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12:9","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43:2","bucket.full-sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26:2","obj_delete_at_hint.0000000116","obj_delete_at_hint.0000000034","obj_delete_at_hint.0000000101","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41:5","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27:8","bucket.full-sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23:0","bucket.full-sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10","obj_delete_at_hint.0000000079","obj_delete_at_hint.0000000049","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21:1","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14:10","obj_delete_at_hint.0000000117","obj_delete_at_hint.0000000044","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19:0","obj_delete_at_hint.0000000066","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1:0","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44:1","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17:0","obj_delete_at_hint.0000000068","obj_delete_at_hint.0000000085","bucket.full-sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1","data_log.0.fPPMMv6vC/7.0","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22:4","bucket.full-sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44:0","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19:3","obj_delete_at_hint.0000000073","obj_delete_at_hint.0000000038","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18:10","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1:10","datalog.sync-status.shard.51055576-6662-409d-8abb-ecaeefd3dd80.0","obj_delete_at_hint.0000000118","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44:2","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25:1","obj_delete_at_hint.0000000036","obj_delete_at_hint.0000000103","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13:3","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15:0","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16:7","obj_delete_at_hint.0000000119","obj_delete_at_hint.0000000098","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44:5","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15:5","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41:7","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26:4","datalog.sync-status.shard.51055576-6662-409d-8abb-ecaeefd3dd80.0.retry","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21:10","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24:2","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43:4","obj_delete_at_hint.0000000027","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13:9","obj_delete_at_hint.0000000019","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14:5","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28:1","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28:7","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27:3","obj_delete_at_hint.0000000039","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20:6","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1:1","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12:0","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16:1","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12:4","obj_delete_at_hint.0000000100","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23:8","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12:7","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28:4","bucket.full-sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24:3","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22:10","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25:0","obj_delete_at_hint.0000000093","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18:4","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41:4","bucket.full-sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26:6","obj_delete_at_hint.0000000004","obj_delete_at_hint.0000000063","obj_delete_at_hint.0000000122","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43:10","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23:3","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23:4","obj_delete_at_hint.0000000057","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1:5","bucket.full-sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21","obj_delete_at_hint.0000000054","obj_delete_at_hint.0000000114","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43:3","data_log.0","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26:9","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25:9","obj_delete_at_hint.0000000012","obj_delete_at_hint.0000000084","obj_delete_at_hint.0000000043","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42:10","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14:2","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3:0:1","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19:6","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13:6","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23:1","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17:10","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12:3","obj_delete_at_hint.0000000111","obj_delete_at_hint.0000000048","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26:8","obj_delete_at_hint.0000000020","obj_delete_at_hint.0000000099","obj_delete_at_hint.0000000056","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16:4","meta.log.cb2fcdce-2c1b-429a-8601-a284ee3a2381.3","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16:10","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23:10","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20:2","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26:7","obj_delete_at_hint.0000000072","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42:0","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25:10","obj_delete_at_hint.0000000062","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17:9","obj_delete_at_hint.0000000115","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16:8","obj_delete_at_hint.0000000033","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20:0","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21:6","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21:9","obj_delete_at_hint.0000000025","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27:0","obj_delete_at_hint.0000000123","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13:7","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15:6","obj_delete_at_hint.0000000108","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42:3","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18:7","bucket.full-sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41","obj_delete_at_hint.0000000094","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18:2","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27:1","obj_delete_at_hint.0000000060","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25:4","obj_delete_at_hint.0000000109","bucket.full-sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23:5","obj_delete_at_hint.0000000013","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15:7","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15:8","obj_delete_at_hint.0000000051","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10:0:1","obj_delete_at_hint.0000000106","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23:2","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18:9","obj_delete_at_hint.0000000017","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42:4","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18:3","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14:4","bucket.full-sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28","obj_delete_at_hint.0000000023","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20:9","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19:4","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14:1","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24:1","obj_delete_at_hint.0000000107","bucket.full-sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20","obj_delete_at_hint.0000000046","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28:2","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24:4","bucket.full-sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14","bucket.full-sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24:7","data_log.2","bucket.full-sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13:10","obj_delete_at_hint.0000000067","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15:2","bucket.full-sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14:8","bucket.full-sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13","_sem_setdata_log.2","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13:1","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12:8","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24:0","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19:9","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41:3","obj_delete_at_hint.0000000086","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1:8","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23:7","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25:6","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20:5","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14:0","_sem_setdata_log.1","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24:8","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28:5","obj_delete_at_hint.0000000050","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13:0","obj_delete_at_hint.0000000102","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42:2","datalog.sync-status.shard.51055576-6662-409d-8abb-ecaeefd3dd80.1.retry","bucket.full-sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17:2","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41:0","obj_delete_at_hint.0000000124","obj_delete_at_hint.0000000001","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20:8","bucket.full-sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16:2","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22:6","obj_delete_at_hint.0000000126","obj_delete_at_hint.0000000055","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16:3","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41:8","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24:10","obj_delete_at_hint.0000000061","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42:7","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16:9","bucket.full-sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24","obj_delete_at_hint.0000000097","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17:8","bucket.full-sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12","obj_delete_at_hint.0000000042","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20:3","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26:1","obj_delete_at_hint.0000000075","obj_delete_at_hint.0000000120","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20:10","obj_delete_at_hint.0000000030","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44:4","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15:1","bucket.sync-status.51055576-6662-409d-8abb-ecaeefd3dd80:dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13:2","meta.log.cb2fcdce-2c1b-429a-8601-a284ee3a2381.2"] 2026-03-20T11:43:59.733 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['log', 'list', '--rgw-zone', 'a2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:59.733 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'log', 'list', '--rgw-zone', 'a2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:59.733 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 log list --rgw-zone a2 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:43:59.810 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:43:59.810 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:43:59.916 INFO:teuthology.orchestra.run.vm02.stdout:["bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19:8","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21:8","obj_delete_at_hint.0000000078","meta.history","obj_delete_at_hint.0000000070","obj_delete_at_hint.0000000104","bucket.full-sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1","obj_delete_at_hint.0000000026","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1:0","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19:9","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1:2","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1:7","obj_delete_at_hint.0000000028","bucket.full-sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13","obj_delete_at_hint.0000000040","bucket.full-sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26:3","obj_delete_at_hint.0000000015","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17:3","obj_delete_at_hint.0000000069","obj_delete_at_hint.0000000095","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41:6","obj_delete_at_hint.0000000003","obj_delete_at_hint.0000000047","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17:1","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27:9","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23:3","obj_delete_at_hint.0000000035","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23:7","bucket.full-sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26","obj_delete_at_hint.0000000037","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44:2","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13:3","obj_delete_at_hint.0000000024","bucket.full-sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28:10","obj_delete_at_hint.0000000000","obj_delete_at_hint.0000000031","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20:7","data_log.3","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21:7","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25:5","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16:0","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21:10","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23:6","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23:1","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44:3","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15:10","data_loggenerations_metadata","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18:7","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24:7","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14:1","obj_delete_at_hint.0000000076","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24:2","obj_delete_at_hint.0000000113","obj_delete_at_hint.0000000005","obj_delete_at_hint.0000000011","obj_delete_at_hint.0000000058","obj_delete_at_hint.0000000052","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42:8","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42:7","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43:2","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14:9","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27:1","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21:9","bucket.full-sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43:3","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17:10","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15:0","obj_delete_at_hint.0000000088","obj_delete_at_hint.0000000080","obj_delete_at_hint.0000000090","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14:8","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26:1","obj_delete_at_hint.0000000110","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22:8","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26:6","obj_delete_at_hint.0000000096","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16:9","_sem_setdata_log.3","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23:8","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13:8","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24:3","obj_delete_at_hint.0000000087","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27:2","obj_delete_at_hint.0000000008","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25:7","meta.log.cb2fcdce-2c1b-429a-8601-a284ee3a2381.0","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16:8","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25:1","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41:7","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14:6","obj_delete_at_hint.0000000006","obj_delete_at_hint.0000000029","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24:1","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12:8","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14:5","obj_delete_at_hint.0000000089","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27:6","datalog.sync-status.shard.4b634ebf-58f5-45bc-839a-cb4b425e773a.1.retry","obj_delete_at_hint.0000000022","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18:0","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15:6","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16:3","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18:4","bilog.trim","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28:5","obj_delete_at_hint.0000000016","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26:8","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10:0:1","obj_delete_at_hint.0000000041","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1:4","obj_delete_at_hint.0000000018","obj_delete_at_hint.0000000092","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18:1","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43:6","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41:0","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26:7","bucket.full-sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23:10","datalog.sync-status.shard.4b634ebf-58f5-45bc-839a-cb4b425e773a.3.retry","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24:6","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28:7","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24:8","obj_delete_at_hint.0000000014","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18:3","_sem_setdata_log.0","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23:5","obj_delete_at_hint.0000000112","obj_delete_at_hint.0000000007","obj_delete_at_hint.0000000021","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12:0","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15:5","obj_delete_at_hint.0000000064","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19:0","obj_delete_at_hint.0000000071","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12:5","obj_delete_at_hint.0000000074","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28:8","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41:5","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18:9","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12:4","obj_delete_at_hint.0000000081","obj_delete_at_hint.0000000009","obj_delete_at_hint.0000000121","data_log.1","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14:2","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3:0:1","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19:7","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21:6","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1:10","obj_delete_at_hint.0000000125","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44:9","obj_delete_at_hint.0000000082","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25:2","obj_delete_at_hint.0000000105","obj_delete_at_hint.0000000059","obj_delete_at_hint.0000000077","meta.log.cb2fcdce-2c1b-429a-8601-a284ee3a2381.1","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42:1","obj_delete_at_hint.0000000032","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17:4","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16:7","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15:4","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25:3","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21:3","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22:1","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27:3","bucket.full-sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12:7","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27:8","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19:10","obj_delete_at_hint.0000000053","obj_delete_at_hint.0000000091","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44:8","datalog.sync-status.shard.4b634ebf-58f5-45bc-839a-cb4b425e773a.2","obj_delete_at_hint.0000000065","bucket.full-sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11","obj_delete_at_hint.0000000083","obj_delete_at_hint.0000000010","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25:0","datalog.sync-status.shard.4b634ebf-58f5-45bc-839a-cb4b425e773a.3","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28:9","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18:2","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13:5","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19:2","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43:5","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21:4","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17:5","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42:5","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20:5","mdlog.sync-status.shard.2","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16:1","obj_delete_at_hint.0000000045","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25:8","datalog.sync-status.shard.4b634ebf-58f5-45bc-839a-cb4b425e773a.0.retry","obj_delete_at_hint.0000000002","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17:0","bucket.full-sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22:9","bucket.full-sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26:9","obj_delete_at_hint.0000000116","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28:0","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22:5","datalog.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a","obj_delete_at_hint.0000000034","obj_delete_at_hint.0000000101","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20:0","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18:8","mdlog.sync-status.shard.1","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41:1","obj_delete_at_hint.0000000079","obj_delete_at_hint.0000000049","obj_delete_at_hint.0000000117","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27:0","data_log.2.RwU3CqmOAcw.0","obj_delete_at_hint.0000000044","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18:10","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25:4","obj_delete_at_hint.0000000066","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41:10","bucket.full-sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41","obj_delete_at_hint.0000000068","obj_delete_at_hint.0000000085","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21:5","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14:3","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17:7","obj_delete_at_hint.0000000073","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14:4","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21:0","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1:9","obj_delete_at_hint.0000000038","obj_delete_at_hint.0000000118","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41:9","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13:4","obj_delete_at_hint.0000000036","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20:2","obj_delete_at_hint.0000000103","bucket.full-sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25","mdlog.sync-status.shard.3","obj_delete_at_hint.0000000119","obj_delete_at_hint.0000000098","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43:0","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18:5","bucket.full-sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27","mdlog.sync-status.shard.0","obj_delete_at_hint.0000000027","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41:2","mdlog.sync-status","obj_delete_at_hint.0000000019","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13:2","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12:2","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12:1","obj_delete_at_hint.0000000039","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22:4","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20:3","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15:2","datalog.sync-status.shard.4b634ebf-58f5-45bc-839a-cb4b425e773a.0","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15:8","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1:6","obj_delete_at_hint.0000000100","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17:6","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25:10","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22:10","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14:0","obj_delete_at_hint.0000000093","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26:5","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17:2","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24:9","obj_delete_at_hint.0000000004","obj_delete_at_hint.0000000063","obj_delete_at_hint.0000000122","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42:10","bucket.full-sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23","obj_delete_at_hint.0000000057","obj_delete_at_hint.0000000054","obj_delete_at_hint.0000000114","bucket.full-sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21","data_log.0","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24:0","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23:4","obj_delete_at_hint.0000000012","obj_delete_at_hint.0000000084","obj_delete_at_hint.0000000043","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19:6","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13:1","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28:6","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44:4","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23:2","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28:4","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43:8","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42:0","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1:1","obj_delete_at_hint.0000000111","obj_delete_at_hint.0000000048","bucket.full-sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44","obj_delete_at_hint.0000000020","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41:3","obj_delete_at_hint.0000000099","obj_delete_at_hint.0000000056","meta.log.cb2fcdce-2c1b-429a-8601-a284ee3a2381.3","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22:7","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13:0","obj_delete_at_hint.0000000072","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22:3","obj_delete_at_hint.0000000062","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19:1","obj_delete_at_hint.0000000115","obj_delete_at_hint.0000000033","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23:0","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16:10","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44:7","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23:9","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13:10","obj_delete_at_hint.0000000025","data_log.1.xtO7a7iWI25.0","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15:9","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16:5","obj_delete_at_hint.0000000123","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22:2","obj_delete_at_hint.0000000108","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13:7","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42:4","obj_delete_at_hint.0000000094","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15:3","data_log.3.Suf0LUjqA3B.0","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26:2","obj_delete_at_hint.0000000060","obj_delete_at_hint.0000000109","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17:9","obj_delete_at_hint.0000000013","bucket.full-sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17","bucket.full-sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20","bucket.full-sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24:10","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20:9","obj_delete_at_hint.0000000051","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42:9","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28:2","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22:0","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24:4","obj_delete_at_hint.0000000106","data_log.0.7lU5G4OpH0n.0","obj_delete_at_hint.0000000017","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20:1","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28:3","obj_delete_at_hint.0000000023","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14:10","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16:4","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43:9","obj_delete_at_hint.0000000107","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44:6","obj_delete_at_hint.0000000046","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43:4","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15:7","bucket.full-sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42:3","datalog.sync-status.shard.4b634ebf-58f5-45bc-839a-cb4b425e773a.2.retry","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13:6","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14:7","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44:5","data_log.2","obj_delete_at_hint.0000000067","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12:6","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43:1","_sem_setdata_log.2","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41:8","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43:10","obj_delete_at_hint.0000000086","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28:1","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22:6","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26:10","datalog.sync-status.shard.4b634ebf-58f5-45bc-839a-cb4b425e773a.1","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20:6","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17:8","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13:9","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44:1","bucket.full-sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16","_sem_setdata_log.1","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1:3","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12:9","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19:3","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25:6","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20:10","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27:7","bucket.full-sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28","obj_delete_at_hint.0000000050","obj_delete_at_hint.0000000102","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1:5","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21:1","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25:9","bucket.full-sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42","obj_delete_at_hint.0000000124","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43:7","obj_delete_at_hint.0000000001","obj_delete_at_hint.0000000126","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27:10","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12:3","obj_delete_at_hint.0000000055","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16:6","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42:2","obj_delete_at_hint.0000000061","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44:0","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20:8","obj_delete_at_hint.0000000097","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21:2","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12:10","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19:5","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11:0:1","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20:4","bucket.full-sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22","obj_delete_at_hint.0000000042","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26:4","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42:6","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27:4","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44:10","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19:4","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24:5","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26:0","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41:4","obj_delete_at_hint.0000000075","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15:1","obj_delete_at_hint.0000000120","obj_delete_at_hint.0000000030","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16:2","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27:5","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1:8","bucket.sync-status.4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18:6","meta.log.cb2fcdce-2c1b-429a-8601-a284ee3a2381.2"] 2026-03-20T11:43:59.932 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:59.932 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:43:59.932 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:44:00.011 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:00.011 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:00.147 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"00000000000000000000:00000000000000003919","last_update":"2026-03-20T11:43:59.524764Z"},{"marker":"00000000000000000000:00000000000000009973","last_update":"2026-03-20T11:43:59.526383Z"},{"marker":"00000000000000000000:00000000000000007302","last_update":"2026-03-20T11:43:59.527734Z"},{"marker":"00000000000000000000:00000000000000005043","last_update":"2026-03-20T11:43:59.529633Z"}] 2026-03-20T11:44:00.148 INFO:rgw_multi.tests:starting data checkpoint for target_zone=a2 source_zone=a1 2026-03-20T11:44:00.148 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:00.148 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:00.148 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --source-zone a1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:44:00.229 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:00.229 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:00.327 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"instance_id":15702199979187085538},"markers":[{"key":0,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000003357","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:28:44.807706Z"}},{"key":1,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000009130","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:35:04.273160Z"}},{"key":2,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000006460","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:36:51.874908Z"}},{"key":3,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000004198","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:35:04.193357Z"}}]},"full_sync":{"total":0,"complete":0},"current_time":"2026-03-20T11:44:00Z"} 2026-03-20T11:44:00.327 WARNING:rgw_multi.tests:data of zone a2 behind zone a1: shard=0 master=00000000000000000000:00000000000000003919 target=00000000000000000000:00000000000000003357, shard=1 master=00000000000000000000:00000000000000009973 target=00000000000000000000:00000000000000009130, shard=2 master=00000000000000000000:00000000000000007302 target=00000000000000000000:00000000000000006460, shard=3 master=00000000000000000000:00000000000000005043 target=00000000000000000000:00000000000000004198 2026-03-20T11:44:01.968 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:44:01.969+0000 7f11de8ad640 -1 WARNING: The OSD cluster is overloaded and struggling to complete ops. You need more capacity to serve this level of demand. 2026-03-20T11:44:05.328 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:05.328 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:05.328 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --source-zone a1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:44:05.373 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:05.373 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:05.474 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"instance_id":15702199979187085538},"markers":[{"key":0,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000003919","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:59.524764Z"}},{"key":1,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000009973","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:59.526383Z"}},{"key":2,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000007302","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:59.527734Z"}},{"key":3,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000005043","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:59.529633Z"}}]},"full_sync":{"total":0,"complete":0},"current_time":"2026-03-20T11:44:05Z"} 2026-03-20T11:44:05.474 INFO:rgw_multi.tests:finished data checkpoint for target_zone=a2 source_zone=a1 2026-03-20T11:44:05.474 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:05.474 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:05.475 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:44:05.516 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:05.516 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:05.619 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"00000000000000000000:00000000000000004201","last_update":"2026-03-20T11:43:59.944245Z"},{"marker":"00000000000000000000:00000000000000010537","last_update":"2026-03-20T11:43:59.945864Z"},{"marker":"00000000000000000000:00000000000000007725","last_update":"2026-03-20T11:43:59.947011Z"},{"marker":"00000000000000000000:00000000000000005467","last_update":"2026-03-20T11:43:59.949057Z"}] 2026-03-20T11:44:05.619 INFO:rgw_multi.tests:starting data checkpoint for target_zone=a1 source_zone=a2 2026-03-20T11:44:05.619 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:05.619 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:05.619 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --source-zone a2 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:44:05.698 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:05.698 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:05.803 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"instance_id":5866003269306126155},"markers":[{"key":0,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000004201","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:59.944245Z"}},{"key":1,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000010537","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:59.945864Z"}},{"key":2,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000007725","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:59.947011Z"}},{"key":3,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000005467","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:59.949057Z"}}]},"full_sync":{"total":0,"complete":0},"current_time":"2026-03-20T11:44:05Z"} 2026-03-20T11:44:05.803 INFO:rgw_multi.tests:finished data checkpoint for target_zone=a1 source_zone=a2 2026-03-20T11:44:05.806 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_sync_disable ... ok 2026-03-20T11:44:05.972 INFO:rgw_multi.tests:create bucket zone=a1 name=dipepu-51 2026-03-20T11:44:05.992 INFO:rgw_multi.tests:create bucket zone=a2 name=dipepu-52 2026-03-20T11:44:06.074 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:06.074 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:06.074 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:44:06.117 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:06.118 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:06.205 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:44:06.215 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774007046.010461_231.1","last_update":"2026-03-20T11:44:06.010461Z"},{"marker":"1_1774007039.336409_210.1","last_update":"2026-03-20T11:43:59.336409Z"}] 2026-03-20T11:44:06.216 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:44:06.216 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:06.216 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:06.216 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:44:06.258 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:06.259 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:06.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:06.347+0000 7f7103410900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:44:06.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:06.348+0000 7f7103410900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:44:06.358 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774007046.010461_231.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:44:06.010461Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774007039.336409_210.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:59.336409Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:44:06Z"} 2026-03-20T11:44:06.358 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:44:06.358 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-51', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:06.358 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-51', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:06.358 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-51 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:44:06.403 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:06.403 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:06.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:06.491+0000 7fda2f220900 1 bucket sync caught up with source: 2026-03-20T11:44:06.489 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000001.35.6, , 00000000001.22.6, , 00000000001.33.6, , , 00000000001.48.6, , , ] 2026-03-20T11:44:06.489 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000001.35.6, , 00000000001.22.6, , 00000000001.33.6, , , 00000000001.48.6, , , ] 2026-03-20T11:44:06.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:06.491+0000 7fda2f220900 0 bucket checkpoint complete 2026-03-20T11:44:06.500 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-51', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:06.500 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-51', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:06.500 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-51 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:44:06.541 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:06.541 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:06.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:06.631+0000 7f87ce52f900 1 waiting to reach incremental sync.. 2026-03-20T11:44:11.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:11.632+0000 7f87ce52f900 1 bucket sync caught up with source: 2026-03-20T11:44:11.630 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000001.56.6, , 00000000001.42.6, , 00000000001.58.6, , , 00000000001.55.6, , , ] 2026-03-20T11:44:11.630 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000001.56.6, , 00000000001.42.6, , 00000000001.58.6, , , 00000000001.55.6, , , ] 2026-03-20T11:44:11.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:11.632+0000 7f87ce52f900 0 bucket checkpoint complete 2026-03-20T11:44:11.640 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-51 zones={a2, a1} 2026-03-20T11:44:11.648 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:44:11.649 DEBUG:tasks.rgw_multi.zone_rados:o=obj1, v=null 2026-03-20T11:44:11.649 DEBUG:tasks.rgw_multi.zone_rados:o=obj2, v=null 2026-03-20T11:44:11.649 DEBUG:tasks.rgw_multi.zone_rados:o=obj3, v=null 2026-03-20T11:44:11.649 DEBUG:tasks.rgw_multi.zone_rados:o=obj4, v=null 2026-03-20T11:44:11.653 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:44:11.653 DEBUG:tasks.rgw_multi.zone_rados:o=obj1, v=null 2026-03-20T11:44:11.653 DEBUG:tasks.rgw_multi.zone_rados:o=obj2, v=null 2026-03-20T11:44:11.653 DEBUG:tasks.rgw_multi.zone_rados:o=obj3, v=null 2026-03-20T11:44:11.653 DEBUG:tasks.rgw_multi.zone_rados:o=obj4, v=null 2026-03-20T11:44:11.653 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2026-03-20T11:44:11.711 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2026-03-20T11:44:11.771 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2026-03-20T11:44:11.831 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2026-03-20T11:44:11.886 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-51 zones={a2, a1} 2026-03-20T11:44:11.886 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-52', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:11.886 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-52', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:11.886 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-52 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:44:11.926 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:11.926 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:12.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:12.009+0000 7f7fb61ca900 1 bucket sync caught up with source: 2026-03-20T11:44:12.007 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000001.15.6, , 00000000001.35.6, , 00000000001.27.6, , , 00000000001.12.6, , , ] 2026-03-20T11:44:12.007 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000001.15.6, , 00000000001.35.6, , 00000000001.27.6, , , 00000000001.12.6, , , ] 2026-03-20T11:44:12.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:12.009+0000 7f7fb61ca900 0 bucket checkpoint complete 2026-03-20T11:44:12.017 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-52', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:12.017 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-52', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:12.017 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-52 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:44:12.102 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:12.102 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:12.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:12.186+0000 7fd095741900 1 bucket sync caught up with source: 2026-03-20T11:44:12.184 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000001.27.6, , 00000000001.55.6, , 00000000001.44.6, , , 00000000001.27.6, , , ] 2026-03-20T11:44:12.184 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000001.27.6, , 00000000001.55.6, , 00000000001.44.6, , , 00000000001.27.6, , , ] 2026-03-20T11:44:12.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:12.186+0000 7fd095741900 0 bucket checkpoint complete 2026-03-20T11:44:12.193 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-52 zones={a2, a1} 2026-03-20T11:44:12.202 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:44:12.202 DEBUG:tasks.rgw_multi.zone_rados:o=obj1, v=null 2026-03-20T11:44:12.202 DEBUG:tasks.rgw_multi.zone_rados:o=obj2, v=null 2026-03-20T11:44:12.202 DEBUG:tasks.rgw_multi.zone_rados:o=obj3, v=null 2026-03-20T11:44:12.202 DEBUG:tasks.rgw_multi.zone_rados:o=obj4, v=null 2026-03-20T11:44:12.205 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:44:12.205 DEBUG:tasks.rgw_multi.zone_rados:o=obj1, v=null 2026-03-20T11:44:12.205 DEBUG:tasks.rgw_multi.zone_rados:o=obj2, v=null 2026-03-20T11:44:12.205 DEBUG:tasks.rgw_multi.zone_rados:o=obj3, v=null 2026-03-20T11:44:12.205 DEBUG:tasks.rgw_multi.zone_rados:o=obj4, v=null 2026-03-20T11:44:12.205 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2026-03-20T11:44:12.264 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2026-03-20T11:44:12.324 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2026-03-20T11:44:12.383 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2026-03-20T11:44:12.444 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-52 zones={a2, a1} 2026-03-20T11:44:12.445 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'disable', '--bucket', 'dipepu-51', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:44:12.445 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'disable', '--bucket', 'dipepu-51', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:44:12.445 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync disable --bucket dipepu-51 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:44:12.481 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:12.481 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:12.609 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'enable', '--bucket', 'dipepu-51', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:44:12.610 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'enable', '--bucket', 'dipepu-51', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:44:12.610 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync enable --bucket dipepu-51 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:44:12.691 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:12.691 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:12.831 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'disable', '--bucket', 'dipepu-52', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:44:12.831 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'disable', '--bucket', 'dipepu-52', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:44:12.831 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync disable --bucket dipepu-52 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:44:12.912 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:12.912 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:13.083 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'enable', '--bucket', 'dipepu-52', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:44:13.083 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'enable', '--bucket', 'dipepu-52', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:44:13.083 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync enable --bucket dipepu-52 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:44:13.166 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:13.166 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:13.352 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-51', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:13.352 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-51', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:13.352 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-51 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:44:13.395 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:13.395 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:13.481 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:13.483+0000 7fa536b2a900 1 waiting for incremental sync to catch up: 2026-03-20T11:44:13.481 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000001.35.6, , 00000000001.22.6, , 00000000001.33.6, , , 00000000001.48.6, , , ] 2026-03-20T11:44:13.481 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000004.41.6, 00000000002.16.1, 00000000004.28.6, 00000000002.34.1, 00000000004.45.6, 00000000002.40.1, 00000000002.39.1, 00000000004.52.6, 00000000002.73.1, 00000000002.41.1, 00000000002.74.1] 2026-03-20T11:44:18.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:18.484+0000 7fa536b2a900 1 bucket sync caught up with source: 2026-03-20T11:44:18.482 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000004.41.6, 00000000002.16.1, 00000000004.28.6, 00000000002.34.1, 00000000004.45.6, 00000000002.40.1, 00000000002.39.1, 00000000004.52.6, 00000000002.73.1, 00000000002.41.1, 00000000002.74.1] 2026-03-20T11:44:18.482 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000004.41.6, 00000000002.16.1, 00000000004.28.6, 00000000002.34.1, 00000000004.45.6, 00000000002.40.1, 00000000002.39.1, 00000000004.52.6, 00000000002.73.1, 00000000002.41.1, 00000000002.74.1] 2026-03-20T11:44:18.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:18.484+0000 7fa536b2a900 0 bucket checkpoint complete 2026-03-20T11:44:18.493 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-51', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:18.493 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-51', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:18.493 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-51 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:44:18.540 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:18.540 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:18.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:18.639+0000 7f6496a15900 1 waiting for incremental sync to catch up: 2026-03-20T11:44:18.638 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000001.56.6, , 00000000001.42.6, , 00000000001.58.6, , , 00000000001.55.6, , , ] 2026-03-20T11:44:18.638 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000002.62.6, , 00000000002.48.6, , 00000000002.70.6, , , 00000000002.59.6, , , ] 2026-03-20T11:44:23.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:23.641+0000 7f6496a15900 1 bucket sync caught up with source: 2026-03-20T11:44:23.639 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000002.62.6, , 00000000002.48.6, , 00000000002.70.6, , , 00000000002.59.6, , , ] 2026-03-20T11:44:23.639 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000002.62.6, , 00000000002.48.6, , 00000000002.70.6, , , 00000000002.59.6, , , ] 2026-03-20T11:44:23.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:23.641+0000 7f6496a15900 0 bucket checkpoint complete 2026-03-20T11:44:23.650 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-51 zones={a2, a1} 2026-03-20T11:44:23.657 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:44:23.657 DEBUG:tasks.rgw_multi.zone_rados:o=obj1, v=null 2026-03-20T11:44:23.657 DEBUG:tasks.rgw_multi.zone_rados:o=obj2, v=null 2026-03-20T11:44:23.657 DEBUG:tasks.rgw_multi.zone_rados:o=obj3, v=null 2026-03-20T11:44:23.657 DEBUG:tasks.rgw_multi.zone_rados:o=obj4, v=null 2026-03-20T11:44:23.657 DEBUG:tasks.rgw_multi.zone_rados:o=obj5, v=null 2026-03-20T11:44:23.657 DEBUG:tasks.rgw_multi.zone_rados:o=obj6, v=null 2026-03-20T11:44:23.657 DEBUG:tasks.rgw_multi.zone_rados:o=obj7, v=null 2026-03-20T11:44:23.657 DEBUG:tasks.rgw_multi.zone_rados:o=obj8, v=null 2026-03-20T11:44:23.660 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:44:23.660 DEBUG:tasks.rgw_multi.zone_rados:o=obj1, v=null 2026-03-20T11:44:23.660 DEBUG:tasks.rgw_multi.zone_rados:o=obj2, v=null 2026-03-20T11:44:23.661 DEBUG:tasks.rgw_multi.zone_rados:o=obj3, v=null 2026-03-20T11:44:23.661 DEBUG:tasks.rgw_multi.zone_rados:o=obj4, v=null 2026-03-20T11:44:23.661 DEBUG:tasks.rgw_multi.zone_rados:o=obj5, v=null 2026-03-20T11:44:23.661 DEBUG:tasks.rgw_multi.zone_rados:o=obj6, v=null 2026-03-20T11:44:23.661 DEBUG:tasks.rgw_multi.zone_rados:o=obj7, v=null 2026-03-20T11:44:23.661 DEBUG:tasks.rgw_multi.zone_rados:o=obj8, v=null 2026-03-20T11:44:23.661 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2026-03-20T11:44:23.719 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2026-03-20T11:44:23.775 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2026-03-20T11:44:23.831 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2026-03-20T11:44:23.890 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj5 2026-03-20T11:44:23.951 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj6 2026-03-20T11:44:24.008 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj7 2026-03-20T11:44:24.067 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj8 2026-03-20T11:44:24.125 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-51 zones={a2, a1} 2026-03-20T11:44:24.125 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-52', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:24.125 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-52', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:24.125 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-52 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:44:24.169 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:24.170 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:24.264 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:24.266+0000 7f6c4ef4f900 1 bucket sync caught up with source: 2026-03-20T11:44:24.264 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000004.19.6, 00000000002.34.1, 00000000004.47.6, 00000000002.22.1, 00000000004.31.6, 00000000002.67.1, 00000000002.32.1, 00000000004.16.6, 00000000002.26.1, 00000000002.76.1, 00000000002.54.1] 2026-03-20T11:44:24.264 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000004.19.6, 00000000002.34.1, 00000000004.47.6, 00000000002.22.1, 00000000004.31.6, 00000000002.67.1, 00000000002.32.1, 00000000004.16.6, 00000000002.26.1, 00000000002.76.1, 00000000002.54.1] 2026-03-20T11:44:24.264 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:24.266+0000 7f6c4ef4f900 0 bucket checkpoint complete 2026-03-20T11:44:24.273 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-52', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:24.273 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-52', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:24.273 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-52 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:44:24.313 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:24.314 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:24.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:24.404+0000 7ff70c125900 1 bucket sync caught up with source: 2026-03-20T11:44:24.402 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000002.30.6, , 00000000002.60.6, , 00000000002.47.6, , , 00000000002.30.6, , , ] 2026-03-20T11:44:24.402 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000002.30.6, , 00000000002.60.6, , 00000000002.47.6, , , 00000000002.30.6, , , ] 2026-03-20T11:44:24.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:24.404+0000 7ff70c125900 0 bucket checkpoint complete 2026-03-20T11:44:24.412 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-52 zones={a2, a1} 2026-03-20T11:44:24.499 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:44:24.499 DEBUG:tasks.rgw_multi.zone_rados:o=obj1, v=null 2026-03-20T11:44:24.499 DEBUG:tasks.rgw_multi.zone_rados:o=obj2, v=null 2026-03-20T11:44:24.499 DEBUG:tasks.rgw_multi.zone_rados:o=obj3, v=null 2026-03-20T11:44:24.499 DEBUG:tasks.rgw_multi.zone_rados:o=obj4, v=null 2026-03-20T11:44:24.499 DEBUG:tasks.rgw_multi.zone_rados:o=obj5, v=null 2026-03-20T11:44:24.499 DEBUG:tasks.rgw_multi.zone_rados:o=obj6, v=null 2026-03-20T11:44:24.499 DEBUG:tasks.rgw_multi.zone_rados:o=obj7, v=null 2026-03-20T11:44:24.499 DEBUG:tasks.rgw_multi.zone_rados:o=obj8, v=null 2026-03-20T11:44:24.504 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:44:24.504 DEBUG:tasks.rgw_multi.zone_rados:o=obj1, v=null 2026-03-20T11:44:24.504 DEBUG:tasks.rgw_multi.zone_rados:o=obj2, v=null 2026-03-20T11:44:24.504 DEBUG:tasks.rgw_multi.zone_rados:o=obj3, v=null 2026-03-20T11:44:24.504 DEBUG:tasks.rgw_multi.zone_rados:o=obj4, v=null 2026-03-20T11:44:24.504 DEBUG:tasks.rgw_multi.zone_rados:o=obj5, v=null 2026-03-20T11:44:24.504 DEBUG:tasks.rgw_multi.zone_rados:o=obj6, v=null 2026-03-20T11:44:24.504 DEBUG:tasks.rgw_multi.zone_rados:o=obj7, v=null 2026-03-20T11:44:24.504 DEBUG:tasks.rgw_multi.zone_rados:o=obj8, v=null 2026-03-20T11:44:24.504 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2026-03-20T11:44:24.560 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2026-03-20T11:44:24.618 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2026-03-20T11:44:24.676 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2026-03-20T11:44:24.735 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj5 2026-03-20T11:44:24.792 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj6 2026-03-20T11:44:24.848 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj7 2026-03-20T11:44:24.904 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj8 2026-03-20T11:44:24.960 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-52 zones={a2, a1} 2026-03-20T11:44:24.960 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:24.960 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:24.960 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:44:24.997 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:24.997 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:25.102 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"00000000000000000000:00000000000000005611","last_update":"2026-03-20T11:44:19.866333Z"},{"marker":"00000000000000000000:00000000000000012793","last_update":"2026-03-20T11:44:19.868882Z"},{"marker":"00000000000000000000:00000000000000008994","last_update":"2026-03-20T11:44:13.282067Z"},{"marker":"00000000000000000000:00000000000000007304","last_update":"2026-03-20T11:44:19.871317Z"}] 2026-03-20T11:44:25.102 INFO:rgw_multi.tests:starting data checkpoint for target_zone=a2 source_zone=a1 2026-03-20T11:44:25.102 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:25.102 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:25.102 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --source-zone a1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:44:25.183 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:25.183 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:25.281 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"instance_id":15702199979187085538},"markers":[{"key":0,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000005611","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:44:19.866333Z"}},{"key":1,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000012793","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:44:19.868882Z"}},{"key":2,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000008994","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:44:13.282067Z"}},{"key":3,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000007304","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:44:19.871317Z"}}]},"full_sync":{"total":0,"complete":0},"current_time":"2026-03-20T11:44:25Z"} 2026-03-20T11:44:25.281 INFO:rgw_multi.tests:finished data checkpoint for target_zone=a2 source_zone=a1 2026-03-20T11:44:25.281 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:25.281 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:25.281 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:44:25.359 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:25.359 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:25.452 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"00000000000000000000:00000000000000004765","last_update":"2026-03-20T11:44:17.899158Z"},{"marker":"00000000000000000000:00000000000000011665","last_update":"2026-03-20T11:44:17.901350Z"},{"marker":"00000000000000000000:00000000000000007725","last_update":"2026-03-20T11:43:59.947011Z"},{"marker":"00000000000000000000:00000000000000006032","last_update":"2026-03-20T11:44:17.902858Z"}] 2026-03-20T11:44:25.452 INFO:rgw_multi.tests:starting data checkpoint for target_zone=a1 source_zone=a2 2026-03-20T11:44:25.452 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:25.452 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:25.452 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --source-zone a2 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:44:25.532 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:25.532 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:25.628 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"instance_id":5866003269306126155},"markers":[{"key":0,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000004765","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:44:17.899158Z"}},{"key":1,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000011665","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:44:17.901350Z"}},{"key":2,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000007725","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:59.947011Z"}},{"key":3,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000006032","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:44:17.902858Z"}}]},"full_sync":{"total":0,"complete":0},"current_time":"2026-03-20T11:44:25Z"} 2026-03-20T11:44:25.628 INFO:rgw_multi.tests:finished data checkpoint for target_zone=a1 source_zone=a2 2026-03-20T11:44:25.632 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_sync_enable_right_after_disable ... ok 2026-03-20T11:44:25.679 INFO:rgw_multi.tests:create bucket zone=a1 name=dipepu-53 2026-03-20T11:44:25.694 INFO:rgw_multi.tests:create bucket zone=a2 name=dipepu-54 2026-03-20T11:44:25.784 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:25.784 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:25.784 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:44:25.829 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:25.829 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:25.923 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:44:25.935 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774007065.712760_239.1","last_update":"2026-03-20T11:44:25.712760Z"},{"marker":"1_1774007039.336409_210.1","last_update":"2026-03-20T11:43:59.336409Z"}] 2026-03-20T11:44:25.935 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:44:25.935 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:25.935 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:25.935 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:44:25.976 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:25.976 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:26.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:26.074+0000 7efc1d5ab900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:44:26.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:26.075+0000 7efc1d5ab900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:44:26.084 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774007065.712760_239.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:44:25.712760Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774007039.336409_210.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:59.336409Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:44:26Z"} 2026-03-20T11:44:26.084 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:44:26.084 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-53', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:26.084 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-53', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:26.084 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-53 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:44:26.166 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:26.166 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:26.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:26.276+0000 7f4faa610900 1 waiting to reach incremental sync.. 2026-03-20T11:44:31.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:31.278+0000 7f4faa610900 1 bucket sync caught up with source: 2026-03-20T11:44:31.276 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000001.58.6, , 00000000001.47.6, , 00000000001.70.6, , , 00000000001.67.6, , , ] 2026-03-20T11:44:31.276 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000001.58.6, , 00000000001.47.6, , 00000000001.70.6, , , 00000000001.67.6, , , ] 2026-03-20T11:44:31.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:31.278+0000 7f4faa610900 0 bucket checkpoint complete 2026-03-20T11:44:31.287 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-53', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:31.287 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-53', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:31.287 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-53 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:44:31.370 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:31.370 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:31.462 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:31.463+0000 7f3a4fd2f900 1 bucket sync caught up with source: 2026-03-20T11:44:31.462 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000001.75.6, , 00000000001.69.6, , 00000000001.83.6, , , 00000000001.84.6, , , ] 2026-03-20T11:44:31.462 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000001.75.6, , 00000000001.69.6, , 00000000001.83.6, , , 00000000001.84.6, , , ] 2026-03-20T11:44:31.462 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:31.464+0000 7f3a4fd2f900 0 bucket checkpoint complete 2026-03-20T11:44:31.474 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-53 zones={a2, a1} 2026-03-20T11:44:31.482 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:44:31.482 DEBUG:tasks.rgw_multi.zone_rados:o=obj1, v=null 2026-03-20T11:44:31.482 DEBUG:tasks.rgw_multi.zone_rados:o=obj2, v=null 2026-03-20T11:44:31.482 DEBUG:tasks.rgw_multi.zone_rados:o=obj3, v=null 2026-03-20T11:44:31.482 DEBUG:tasks.rgw_multi.zone_rados:o=obj4, v=null 2026-03-20T11:44:31.488 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:44:31.488 DEBUG:tasks.rgw_multi.zone_rados:o=obj1, v=null 2026-03-20T11:44:31.488 DEBUG:tasks.rgw_multi.zone_rados:o=obj2, v=null 2026-03-20T11:44:31.488 DEBUG:tasks.rgw_multi.zone_rados:o=obj3, v=null 2026-03-20T11:44:31.488 DEBUG:tasks.rgw_multi.zone_rados:o=obj4, v=null 2026-03-20T11:44:31.488 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2026-03-20T11:44:31.548 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2026-03-20T11:44:31.608 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2026-03-20T11:44:31.671 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2026-03-20T11:44:31.798 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-53 zones={a2, a1} 2026-03-20T11:44:31.798 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-54', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:31.798 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-54', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:31.798 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-54 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:44:31.840 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:31.840 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:31.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:31.931+0000 7f5df4b2a900 1 bucket sync caught up with source: 2026-03-20T11:44:31.929 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000001.42.6, , 00000000001.56.6, , 00000000001.14.6, , , 00000000001.60.6, , , ] 2026-03-20T11:44:31.929 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000001.42.6, , 00000000001.56.6, , 00000000001.14.6, , , 00000000001.60.6, , , ] 2026-03-20T11:44:31.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:31.931+0000 7f5df4b2a900 0 bucket checkpoint complete 2026-03-20T11:44:31.940 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-54', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:31.940 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-54', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:31.940 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-54 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:44:32.022 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:32.022 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:32.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:32.109+0000 7f18d0737900 1 bucket sync caught up with source: 2026-03-20T11:44:32.107 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000001.59.6, , 00000000001.89.6, , 00000000001.35.6, , , 00000000001.71.6, , , ] 2026-03-20T11:44:32.107 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000001.59.6, , 00000000001.89.6, , 00000000001.35.6, , , 00000000001.71.6, , , ] 2026-03-20T11:44:32.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:32.109+0000 7f18d0737900 0 bucket checkpoint complete 2026-03-20T11:44:32.117 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-54 zones={a2, a1} 2026-03-20T11:44:32.124 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:44:32.124 DEBUG:tasks.rgw_multi.zone_rados:o=obj1, v=null 2026-03-20T11:44:32.124 DEBUG:tasks.rgw_multi.zone_rados:o=obj2, v=null 2026-03-20T11:44:32.124 DEBUG:tasks.rgw_multi.zone_rados:o=obj3, v=null 2026-03-20T11:44:32.124 DEBUG:tasks.rgw_multi.zone_rados:o=obj4, v=null 2026-03-20T11:44:32.127 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:44:32.127 DEBUG:tasks.rgw_multi.zone_rados:o=obj1, v=null 2026-03-20T11:44:32.127 DEBUG:tasks.rgw_multi.zone_rados:o=obj2, v=null 2026-03-20T11:44:32.127 DEBUG:tasks.rgw_multi.zone_rados:o=obj3, v=null 2026-03-20T11:44:32.127 DEBUG:tasks.rgw_multi.zone_rados:o=obj4, v=null 2026-03-20T11:44:32.127 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2026-03-20T11:44:32.182 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2026-03-20T11:44:32.243 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2026-03-20T11:44:32.299 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2026-03-20T11:44:32.355 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-54 zones={a2, a1} 2026-03-20T11:44:32.355 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'disable', '--bucket', 'dipepu-53', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:44:32.355 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'disable', '--bucket', 'dipepu-53', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:44:32.355 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync disable --bucket dipepu-53 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:44:32.390 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:32.390 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:32.500 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'disable', '--bucket', 'dipepu-54', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:44:32.500 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'disable', '--bucket', 'dipepu-54', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:44:32.500 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync disable --bucket dipepu-54 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:44:32.577 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:32.577 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:32.693 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:32.693 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:32.693 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:44:32.772 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:32.772 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:32.848 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:44:32.859 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774007072.684957_241.1","last_update":"2026-03-20T11:44:32.684957Z"},{"marker":"1_1774007039.336409_210.1","last_update":"2026-03-20T11:43:59.336409Z"}] 2026-03-20T11:44:32.859 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:44:32.859 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:32.859 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:32.859 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:44:32.936 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:32.936 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:33.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:33.006+0000 7fb1a5f26900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:44:33.004 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:33.006+0000 7fb1a5f26900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:44:33.014 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774007065.712760_239.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:44:25.712760Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774007039.336409_210.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:59.336409Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:44:33Z"} 2026-03-20T11:44:33.014 WARNING:rgw_multi.tests:zone a2 behind master: shard=2 master=1_1774007072.684957_241.1 target=1_1774007065.712760_239.1 2026-03-20T11:44:37.422 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:44:37.424+0000 7f3ddd7a1640 -1 WARNING: The OSD cluster is overloaded and struggling to complete ops. You need more capacity to serve this level of demand. 2026-03-20T11:44:38.015 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:38.015 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:44:38.015 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:44:38.056 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:38.057 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:38.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:38.137+0000 7f56c6210900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:44:38.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:38.138+0000 7f56c6210900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:44:38.148 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774007072.684957_241.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:44:32.684957Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774007039.336409_210.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:59.336409Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:44:38Z"} 2026-03-20T11:44:38.149 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:44:38.188 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'enable', '--bucket', 'dipepu-53', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:44:38.188 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'enable', '--bucket', 'dipepu-53', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:44:38.188 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync enable --bucket dipepu-53 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:44:38.226 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:38.226 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:38.362 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'enable', '--bucket', 'dipepu-54', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:44:38.362 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'enable', '--bucket', 'dipepu-54', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:44:38.362 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync enable --bucket dipepu-54 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:44:38.440 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:38.441 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:38.560 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-53', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:38.560 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-53', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:38.560 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-53 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:44:38.642 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:38.642 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:38.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:38.720+0000 7fefcb92c900 1 waiting to reach incremental sync.. 2026-03-20T11:44:43.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:43.722+0000 7fefcb92c900 1 bucket sync caught up with source: 2026-03-20T11:44:43.720 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000004.65.1, 00000000002.83.1, 00000000004.51.1, 00000000002.21.1, 00000000004.74.1, 00000000002.72.1, 00000000002.84.1, 00000000004.73.1, 00000000002.22.1, 00000000002.50.1, 00000000002.35.1] 2026-03-20T11:44:43.720 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000004.65.1, 00000000002.83.1, 00000000004.51.1, 00000000002.21.1, 00000000004.74.1, 00000000002.72.1, 00000000002.84.1, 00000000004.73.1, 00000000002.22.1, 00000000002.50.1, 00000000002.35.1] 2026-03-20T11:44:43.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:43.722+0000 7fefcb92c900 0 bucket checkpoint complete 2026-03-20T11:44:43.731 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-53', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:43.731 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-53', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:43.731 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-53 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:44:43.813 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:43.813 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:43.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:43.906+0000 7f79be52e900 1 bucket sync caught up with source: 2026-03-20T11:44:43.904 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000004.87.6, 00000000002.126.1, 00000000004.75.6, 00000000002.34.1, 00000000004.89.6, 00000000002.87.1, 00000000002.128.1, 00000000004.94.6, 00000000002.37.1, 00000000002.73.1, 00000000002.61.1] 2026-03-20T11:44:43.904 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000004.87.6, 00000000002.126.1, 00000000004.75.6, 00000000002.34.1, 00000000004.89.6, 00000000002.87.1, 00000000002.128.1, 00000000004.94.6, 00000000002.37.1, 00000000002.73.1, 00000000002.61.1] 2026-03-20T11:44:43.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:43.906+0000 7f79be52e900 0 bucket checkpoint complete 2026-03-20T11:44:43.915 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-53 zones={a2, a1} 2026-03-20T11:44:43.922 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:44:43.922 DEBUG:tasks.rgw_multi.zone_rados:o=obj1, v=null 2026-03-20T11:44:43.922 DEBUG:tasks.rgw_multi.zone_rados:o=obj2, v=null 2026-03-20T11:44:43.922 DEBUG:tasks.rgw_multi.zone_rados:o=obj3, v=null 2026-03-20T11:44:43.922 DEBUG:tasks.rgw_multi.zone_rados:o=obj4, v=null 2026-03-20T11:44:43.922 DEBUG:tasks.rgw_multi.zone_rados:o=obj5, v=null 2026-03-20T11:44:43.922 DEBUG:tasks.rgw_multi.zone_rados:o=obj6, v=null 2026-03-20T11:44:43.922 DEBUG:tasks.rgw_multi.zone_rados:o=obj7, v=null 2026-03-20T11:44:43.922 DEBUG:tasks.rgw_multi.zone_rados:o=obj8, v=null 2026-03-20T11:44:43.925 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:44:43.925 DEBUG:tasks.rgw_multi.zone_rados:o=obj1, v=null 2026-03-20T11:44:43.925 DEBUG:tasks.rgw_multi.zone_rados:o=obj2, v=null 2026-03-20T11:44:43.925 DEBUG:tasks.rgw_multi.zone_rados:o=obj3, v=null 2026-03-20T11:44:43.925 DEBUG:tasks.rgw_multi.zone_rados:o=obj4, v=null 2026-03-20T11:44:43.925 DEBUG:tasks.rgw_multi.zone_rados:o=obj5, v=null 2026-03-20T11:44:43.925 DEBUG:tasks.rgw_multi.zone_rados:o=obj6, v=null 2026-03-20T11:44:43.925 DEBUG:tasks.rgw_multi.zone_rados:o=obj7, v=null 2026-03-20T11:44:43.925 DEBUG:tasks.rgw_multi.zone_rados:o=obj8, v=null 2026-03-20T11:44:43.925 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2026-03-20T11:44:43.983 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2026-03-20T11:44:44.046 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2026-03-20T11:44:44.104 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2026-03-20T11:44:44.162 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj5 2026-03-20T11:44:44.221 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj6 2026-03-20T11:44:44.278 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj7 2026-03-20T11:44:44.330 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj8 2026-03-20T11:44:44.382 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-53 zones={a2, a1} 2026-03-20T11:44:44.382 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-54', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:44.382 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-54', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:44:44.383 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-54 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:44:44.424 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:44:44.425 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:44:44.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:44.510+0000 7fc56d936900 1 waiting for incremental sync to catch up: 2026-03-20T11:44:44.508 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000004.46.6, 00000000002.37.1, 00000000004.60.6, 00000000002.19.1, 00000000004.18.6, 00000000002.57.1, 00000000002.32.1, 00000000003.66.1, 00000000002.43.1, 00000000002.85.1, 00000000002.30.1] 2026-03-20T11:44:44.508 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000004.46.6, 00000000002.37.1, 00000000004.60.6, 00000000002.19.1, 00000000004.18.6, 00000000002.57.1, 00000000002.32.1, 00000000004.68.6, 00000000002.43.1, 00000000002.85.1, 00000000002.30.1] 2026-03-20T11:44:49.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:49.511+0000 7fc56d936900 1 waiting for incremental sync to catch up: 2026-03-20T11:44:49.510 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000004.46.6, 00000000002.37.1, 00000000004.60.6, 00000000002.19.1, 00000000004.18.6, 00000000002.57.1, 00000000002.32.1, 00000000003.66.1, 00000000002.43.1, 00000000002.85.1, 00000000002.30.1] 2026-03-20T11:44:49.510 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000004.46.6, 00000000002.37.1, 00000000004.60.6, 00000000002.19.1, 00000000004.18.6, 00000000002.57.1, 00000000002.32.1, 00000000004.68.6, 00000000002.43.1, 00000000002.85.1, 00000000002.30.1] 2026-03-20T11:44:54.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:54.513+0000 7fc56d936900 1 waiting for incremental sync to catch up: 2026-03-20T11:44:54.511 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000004.46.6, 00000000002.37.1, 00000000004.60.6, 00000000002.19.1, 00000000004.18.6, 00000000002.57.1, 00000000002.32.1, 00000000003.66.1, 00000000002.43.1, 00000000002.85.1, 00000000002.30.1] 2026-03-20T11:44:54.511 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000004.46.6, 00000000002.37.1, 00000000004.60.6, 00000000002.19.1, 00000000004.18.6, 00000000002.57.1, 00000000002.32.1, 00000000004.68.6, 00000000002.43.1, 00000000002.85.1, 00000000002.30.1] 2026-03-20T11:44:59.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:44:59.514+0000 7fc56d936900 1 waiting for incremental sync to catch up: 2026-03-20T11:44:59.512 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000004.46.6, 00000000002.37.1, 00000000004.60.6, 00000000002.19.1, 00000000004.18.6, 00000000002.57.1, 00000000002.32.1, 00000000003.66.1, 00000000002.43.1, 00000000002.85.1, 00000000002.30.1] 2026-03-20T11:44:59.512 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000004.46.6, 00000000002.37.1, 00000000004.60.6, 00000000002.19.1, 00000000004.18.6, 00000000002.57.1, 00000000002.32.1, 00000000004.68.6, 00000000002.43.1, 00000000002.85.1, 00000000002.30.1] 2026-03-20T11:45:04.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:04.515+0000 7fc56d936900 1 waiting for incremental sync to catch up: 2026-03-20T11:45:04.513 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000004.46.6, 00000000002.37.1, 00000000004.60.6, 00000000002.19.1, 00000000004.18.6, 00000000002.57.1, 00000000002.32.1, 00000000003.66.1, 00000000002.43.1, 00000000002.85.1, 00000000002.30.1] 2026-03-20T11:45:04.513 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000004.46.6, 00000000002.37.1, 00000000004.60.6, 00000000002.19.1, 00000000004.18.6, 00000000002.57.1, 00000000002.32.1, 00000000004.68.6, 00000000002.43.1, 00000000002.85.1, 00000000002.30.1] 2026-03-20T11:45:09.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:09.516+0000 7fc56d936900 1 bucket sync caught up with source: 2026-03-20T11:45:09.514 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000004.46.6, 00000000002.37.1, 00000000004.60.6, 00000000002.19.1, 00000000004.18.6, 00000000002.57.1, 00000000002.32.1, 00000000004.68.6, 00000000002.43.1, 00000000002.85.1, 00000000002.30.1] 2026-03-20T11:45:09.514 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000004.46.6, 00000000002.37.1, 00000000004.60.6, 00000000002.19.1, 00000000004.18.6, 00000000002.57.1, 00000000002.32.1, 00000000004.68.6, 00000000002.43.1, 00000000002.85.1, 00000000002.30.1] 2026-03-20T11:45:09.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:09.516+0000 7fc56d936900 0 bucket checkpoint complete 2026-03-20T11:45:09.523 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-54', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:45:09.523 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-54', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:45:09.523 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-54 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:45:09.563 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:09.563 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:09.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:09.647+0000 7f8785f23900 1 bucket sync caught up with source: 2026-03-20T11:45:09.645 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000004.65.1, 00000000002.57.1, 00000000004.95.1, 00000000002.36.1, 00000000004.41.1, 00000000002.67.1, 00000000002.53.1, 00000000004.82.1, 00000000002.67.1, 00000000002.123.1, 00000000002.39.1] 2026-03-20T11:45:09.645 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000004.65.1, 00000000002.57.1, 00000000004.95.1, 00000000002.36.1, 00000000004.41.1, 00000000002.67.1, 00000000002.53.1, 00000000004.82.1, 00000000002.67.1, 00000000002.123.1, 00000000002.39.1] 2026-03-20T11:45:09.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:09.647+0000 7f8785f23900 0 bucket checkpoint complete 2026-03-20T11:45:09.657 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-54 zones={a2, a1} 2026-03-20T11:45:09.663 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:45:09.664 DEBUG:tasks.rgw_multi.zone_rados:o=obj1, v=null 2026-03-20T11:45:09.664 DEBUG:tasks.rgw_multi.zone_rados:o=obj2, v=null 2026-03-20T11:45:09.664 DEBUG:tasks.rgw_multi.zone_rados:o=obj3, v=null 2026-03-20T11:45:09.664 DEBUG:tasks.rgw_multi.zone_rados:o=obj4, v=null 2026-03-20T11:45:09.664 DEBUG:tasks.rgw_multi.zone_rados:o=obj5, v=null 2026-03-20T11:45:09.664 DEBUG:tasks.rgw_multi.zone_rados:o=obj6, v=null 2026-03-20T11:45:09.664 DEBUG:tasks.rgw_multi.zone_rados:o=obj7, v=null 2026-03-20T11:45:09.664 DEBUG:tasks.rgw_multi.zone_rados:o=obj8, v=null 2026-03-20T11:45:09.670 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:45:09.670 DEBUG:tasks.rgw_multi.zone_rados:o=obj1, v=null 2026-03-20T11:45:09.670 DEBUG:tasks.rgw_multi.zone_rados:o=obj2, v=null 2026-03-20T11:45:09.670 DEBUG:tasks.rgw_multi.zone_rados:o=obj3, v=null 2026-03-20T11:45:09.670 DEBUG:tasks.rgw_multi.zone_rados:o=obj4, v=null 2026-03-20T11:45:09.670 DEBUG:tasks.rgw_multi.zone_rados:o=obj5, v=null 2026-03-20T11:45:09.670 DEBUG:tasks.rgw_multi.zone_rados:o=obj6, v=null 2026-03-20T11:45:09.670 DEBUG:tasks.rgw_multi.zone_rados:o=obj7, v=null 2026-03-20T11:45:09.670 DEBUG:tasks.rgw_multi.zone_rados:o=obj8, v=null 2026-03-20T11:45:09.670 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj1 2026-03-20T11:45:09.730 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj2 2026-03-20T11:45:09.786 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj3 2026-03-20T11:45:09.842 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj4 2026-03-20T11:45:09.900 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj5 2026-03-20T11:45:09.958 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj6 2026-03-20T11:45:10.015 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj7 2026-03-20T11:45:10.071 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj8 2026-03-20T11:45:10.132 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-54 zones={a2, a1} 2026-03-20T11:45:10.133 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:10.133 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:10.133 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:45:10.171 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:10.171 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:10.274 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"00000000000000000000:00000000000000007303","last_update":"2026-03-20T11:45:03.886627Z"},{"marker":"00000000000000000000:00000000000000015613","last_update":"2026-03-20T11:45:03.888250Z"},{"marker":"00000000000000000000:00000000000000010686","last_update":"2026-03-20T11:44:38.545931Z"},{"marker":"00000000000000000000:00000000000000009564","last_update":"2026-03-20T11:45:03.890221Z"}] 2026-03-20T11:45:10.274 INFO:rgw_multi.tests:starting data checkpoint for target_zone=a2 source_zone=a1 2026-03-20T11:45:10.274 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:10.274 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:10.275 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --source-zone a1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:45:10.353 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:10.353 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:10.445 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"instance_id":15702199979187085538},"markers":[{"key":0,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000007303","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:03.886627Z"}},{"key":1,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000015613","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:03.888250Z"}},{"key":2,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000010686","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:44:38.545931Z"}},{"key":3,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000009564","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:03.890221Z"}}]},"full_sync":{"total":0,"complete":0},"current_time":"2026-03-20T11:45:10Z"} 2026-03-20T11:45:10.445 INFO:rgw_multi.tests:finished data checkpoint for target_zone=a2 source_zone=a1 2026-03-20T11:45:10.445 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:10.445 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:10.445 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:45:10.524 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:10.524 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:10.632 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"00000000000000000000:00000000000000006034","last_update":"2026-03-20T11:45:01.921002Z"},{"marker":"00000000000000000000:00000000000000013639","last_update":"2026-03-20T11:45:01.922957Z"},{"marker":"00000000000000000000:00000000000000009417","last_update":"2026-03-20T11:44:39.912037Z"},{"marker":"00000000000000000000:00000000000000007869","last_update":"2026-03-20T11:45:01.924327Z"}] 2026-03-20T11:45:10.632 INFO:rgw_multi.tests:starting data checkpoint for target_zone=a1 source_zone=a2 2026-03-20T11:45:10.632 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:10.632 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:10.632 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --source-zone a2 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:45:10.713 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:10.713 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:10.808 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"instance_id":5866003269306126155},"markers":[{"key":0,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000006034","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:01.921002Z"}},{"key":1,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000013639","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:01.922957Z"}},{"key":2,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000009417","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:44:39.912037Z"}},{"key":3,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000007869","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:01.924327Z"}}]},"full_sync":{"total":0,"complete":0},"current_time":"2026-03-20T11:45:10Z"} 2026-03-20T11:45:10.808 INFO:rgw_multi.tests:finished data checkpoint for target_zone=a1 source_zone=a2 2026-03-20T11:45:10.811 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_sync_disable_enable ... ok 2026-03-20T11:45:10.961 INFO:rgw_multi.tests:create bucket zone=a1 name=dipepu-55 2026-03-20T11:45:10.977 INFO:rgw_multi.tests:create bucket zone=a2 name=dipepu-56 2026-03-20T11:45:12.394 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:12.394 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:12.395 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:45:12.432 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:12.433 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:12.513 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:45:12.525 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774007110.994743_247.1","last_update":"2026-03-20T11:45:10.994743Z"},{"marker":"1_1774007039.336409_210.1","last_update":"2026-03-20T11:43:59.336409Z"}] 2026-03-20T11:45:12.525 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:45:12.525 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:12.525 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:12.525 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:45:12.605 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:12.605 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:12.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:12.695+0000 7f5209d32900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:45:12.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:12.696+0000 7f5209d32900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:45:12.705 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774007110.994743_247.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:10.994743Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774007039.336409_210.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:59.336409Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:45:12Z"} 2026-03-20T11:45:12.705 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:45:12.705 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-55', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:45:12.705 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-55', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:45:12.705 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-55 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:45:12.790 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:12.790 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:12.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:12.880+0000 7f3e8320f900 1 waiting to reach incremental sync.. 2026-03-20T11:45:17.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:17.882+0000 7f3e8320f900 1 bucket sync caught up with source: 2026-03-20T11:45:17.880 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000006.98.6, , , , , ] 2026-03-20T11:45:17.880 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000006.98.6, , , , , ] 2026-03-20T11:45:17.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:17.882+0000 7f3e8320f900 0 bucket checkpoint complete 2026-03-20T11:45:17.890 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-55', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:45:17.890 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-55', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:45:17.890 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-55 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:45:17.932 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:17.932 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:18.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:18.034+0000 7f0699210900 1 bucket sync caught up with source: 2026-03-20T11:45:18.032 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , , , , 00000000001.133.6, , , , , ] 2026-03-20T11:45:18.032 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, , , , , 00000000001.133.6, , , , , ] 2026-03-20T11:45:18.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:18.034+0000 7f0699210900 0 bucket checkpoint complete 2026-03-20T11:45:18.047 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-55 zones={a2, a1} 2026-03-20T11:45:18.057 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:45:18.057 DEBUG:tasks.rgw_multi.zone_rados:o=MULTIPART, v=null 2026-03-20T11:45:18.061 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:45:18.061 DEBUG:tasks.rgw_multi.zone_rados:o=MULTIPART, v=null 2026-03-20T11:45:18.061 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=MULTIPART 2026-03-20T11:45:18.134 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-55 zones={a2, a1} 2026-03-20T11:45:18.137 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_multipart_object_sync ... ok 2026-03-20T11:45:18.193 INFO:rgw_multi.tests:create bucket zone=a1 name=dipepu-57 2026-03-20T11:45:18.238 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:18.238 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:18.238 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:45:18.277 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:18.277 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:18.371 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:45:18.386 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774007118.218776_249.1","last_update":"2026-03-20T11:45:18.218776Z"},{"marker":"1_1774007039.336409_210.1","last_update":"2026-03-20T11:43:59.336409Z"}] 2026-03-20T11:45:18.387 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:45:18.387 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:18.387 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:18.387 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:45:18.471 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:18.471 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:18.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:18.559+0000 7fbc94fcc900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:45:18.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:18.560+0000 7fbc94fcc900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:45:18.569 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774007118.218776_249.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:18.218776Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774007039.336409_210.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:59.336409Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:45:18Z"} 2026-03-20T11:45:18.570 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:45:18.570 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-57', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:45:18.570 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-57', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:45:18.570 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-57 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:45:18.610 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:18.610 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:18.700 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:18.702+0000 7f8816813900 1 bucket sync caught up with source: 2026-03-20T11:45:18.700 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, 00000000001.41.6, , , , 00000000001.23.6, , , , , ] 2026-03-20T11:45:18.700 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, 00000000001.41.6, , , , 00000000001.23.6, , , , , ] 2026-03-20T11:45:18.700 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:18.702+0000 7f8816813900 0 bucket checkpoint complete 2026-03-20T11:45:18.769 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_encrypted_object_sync ... ok 2026-03-20T11:45:18.902 INFO:rgw_multi.tests:create bucket zone=a1 name=dipepu-58 2026-03-20T11:45:18.960 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:18.960 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:18.960 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:45:18.999 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:19.000 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:19.741 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:45:19.778 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774007118.922331_251.1","last_update":"2026-03-20T11:45:18.922331Z"},{"marker":"1_1774007039.336409_210.1","last_update":"2026-03-20T11:43:59.336409Z"}] 2026-03-20T11:45:19.779 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:45:19.779 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:19.779 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:19.779 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:45:19.859 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:19.859 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:19.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:19.948+0000 7f8692215900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:45:19.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:19.949+0000 7f8692215900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:45:19.960 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774007118.922331_251.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:18.922331Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774007039.336409_210.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:59.336409Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:45:19Z"} 2026-03-20T11:45:19.960 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:45:19.960 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-58', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:45:19.960 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-58', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:45:19.960 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-58 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:45:20.045 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:20.045 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:20.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:20.138+0000 7fd055b2a900 1 waiting to reach incremental sync.. 2026-03-20T11:45:25.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:25.140+0000 7fd055b2a900 1 bucket sync caught up with source: 2026-03-20T11:45:25.138 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, 00000000001.77.6, , , , 00000000001.65.6, , , 00000000001.20.6, , 00000000001.28.6] 2026-03-20T11:45:25.138 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, 00000000001.77.6, , , , 00000000001.65.6, , , 00000000001.20.6, , 00000000001.28.6] 2026-03-20T11:45:25.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:25.140+0000 7fd055b2a900 0 bucket checkpoint complete 2026-03-20T11:45:25.150 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-58', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:45:25.150 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-58', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:45:25.150 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-58 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:45:25.192 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:25.192 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:25.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:25.292+0000 7f2772386900 1 waiting to reach incremental sync.. 2026-03-20T11:45:30.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.294+0000 7f2772386900 1 bucket sync caught up with source: 2026-03-20T11:45:30.292 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, 00000000001.98.6, , , , 00000000001.102.6, , , 00000000001.40.6, , 00000000001.48.6] 2026-03-20T11:45:30.292 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, 00000000001.98.6, , , , 00000000001.102.6, , , 00000000001.40.6, , 00000000001.48.6] 2026-03-20T11:45:30.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.294+0000 7f2772386900 0 bucket checkpoint complete 2026-03-20T11:45:30.302 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-58 zones={a2, a1} 2026-03-20T11:45:30.310 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:45:30.310 DEBUG:tasks.rgw_multi.zone_rados:o=a, v=null 2026-03-20T11:45:30.310 DEBUG:tasks.rgw_multi.zone_rados:o=b, v=null 2026-03-20T11:45:30.310 DEBUG:tasks.rgw_multi.zone_rados:o=c, v=null 2026-03-20T11:45:30.310 DEBUG:tasks.rgw_multi.zone_rados:o=d, v=null 2026-03-20T11:45:30.313 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:45:30.313 DEBUG:tasks.rgw_multi.zone_rados:o=a, v=null 2026-03-20T11:45:30.313 DEBUG:tasks.rgw_multi.zone_rados:o=b, v=null 2026-03-20T11:45:30.313 DEBUG:tasks.rgw_multi.zone_rados:o=c, v=null 2026-03-20T11:45:30.313 DEBUG:tasks.rgw_multi.zone_rados:o=d, v=null 2026-03-20T11:45:30.313 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2026-03-20T11:45:30.367 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2026-03-20T11:45:30.418 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2026-03-20T11:45:30.470 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2026-03-20T11:45:30.522 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-58 zones={a2, a1} 2026-03-20T11:45:30.522 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '0', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:45:30.522 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '0', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:45:30.522 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bilog autotrim --rgw-sync-log-trim-max-buckets 0 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 20 --debug-ms 0 2026-03-20T11:45:30.559 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:30.560 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:30.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.576+0000 7fdbb1210900 20 rados->read ofs=0 len=0 2026-03-20T11:45:30.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.577+0000 7fdbb1210900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:45:30.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.577+0000 7fdbb1210900 20 rados->read ofs=0 len=0 2026-03-20T11:45:30.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.577+0000 7fdbb1210900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:45:30.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.577+0000 7fdbb1210900 20 realm test-realm 6a789b89-c2b8-45f5-82cc-fb80234d0c6c 2026-03-20T11:45:30.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.577+0000 7fdbb1210900 20 rados->read ofs=0 len=0 2026-03-20T11:45:30.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.577+0000 7fdbb1210900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:45:30.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.577+0000 7fdbb1210900 20 rados->read ofs=0 len=0 2026-03-20T11:45:30.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.578+0000 7fdbb1210900 20 rados_obj.operate() r=0 bl.length=10 2026-03-20T11:45:30.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.578+0000 7fdbb1210900 20 rados->read ofs=0 len=0 2026-03-20T11:45:30.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.578+0000 7fdbb1210900 20 rados_obj.operate() r=0 bl.length=2020 2026-03-20T11:45:30.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.578+0000 7fdbb1210900 20 rados->read ofs=0 len=0 2026-03-20T11:45:30.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.579+0000 7fdbb1210900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:45:30.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.579+0000 7fdbb1210900 20 rados->read ofs=0 len=0 2026-03-20T11:45:30.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.579+0000 7fdbb1210900 20 rados_obj.operate() r=0 bl.length=1030 2026-03-20T11:45:30.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.579+0000 7fdbb1210900 20 zone a1 found 2026-03-20T11:45:30.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.579+0000 7fdbb1210900 4 Realm: test-realm (6a789b89-c2b8-45f5-82cc-fb80234d0c6c) 2026-03-20T11:45:30.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.579+0000 7fdbb1210900 4 ZoneGroup: a (a) 2026-03-20T11:45:30.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.579+0000 7fdbb1210900 4 Zone: a1 (4b634ebf-58f5-45bc-839a-cb4b425e773a) 2026-03-20T11:45:30.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.579+0000 7fdbb1210900 4 using period configuration: cb2fcdce-2c1b-429a-8601-a284ee3a2381:12 2026-03-20T11:45:30.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.579+0000 7fdbb1210900 20 period zonegroup name a 2026-03-20T11:45:30.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.579+0000 7fdbb1210900 20 using current period zonegroup a 2026-03-20T11:45:30.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.579+0000 7fdbb1210900 20 rados->read ofs=0 len=0 2026-03-20T11:45:30.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.580+0000 7fdbb1210900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:30.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.580+0000 7fdbb1210900 20 rados->read ofs=0 len=0 2026-03-20T11:45:30.578 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.580+0000 7fdbb1210900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:30.578 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.580+0000 7fdbb1210900 20 init(): pipe_map (zone=4b634ebf-58f5-45bc-839a-cb4b425e773a bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:45:30.578 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.580+0000 7fdbb1210900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:30.578 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.580+0000 7fdbb1210900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:30.578 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.580+0000 7fdbb1210900 20 rados->read ofs=0 len=0 2026-03-20T11:45:30.578 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.580+0000 7fdbb1210900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:30.578 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.580+0000 7fdbb1210900 20 rados->read ofs=0 len=0 2026-03-20T11:45:30.578 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.580+0000 7fdbb1210900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:30.578 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.580+0000 7fdbb1210900 20 init(): pipe_map (zone=51055576-6662-409d-8abb-ecaeefd3dd80 bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:45:30.578 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.580+0000 7fdbb1210900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:30.578 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.580+0000 7fdbb1210900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:30.578 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.580+0000 7fdbb1210900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:30.578 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.580+0000 7fdbb1210900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:30.578 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.580+0000 7fdbb1210900 20 started sync module instance, tier type = 2026-03-20T11:45:30.578 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.580+0000 7fdbb1210900 20 generating connection object for zone a2 id 51055576-6662-409d-8abb-ecaeefd3dd80 2026-03-20T11:45:30.578 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.580+0000 7fdbb1210900 20 started zone id=4b634ebf-58f5-45bc-839a-cb4b425e773a (name=a1) with tier type = 2026-03-20T11:45:30.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.584+0000 7fdbb1210900 20 add_watcher() i=6 2026-03-20T11:45:30.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.584+0000 7fdbb1210900 20 add_watcher() i=5 2026-03-20T11:45:30.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.584+0000 7fdbb1210900 20 add_watcher() i=1 2026-03-20T11:45:30.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.585+0000 7fdbb1210900 20 add_watcher() i=4 2026-03-20T11:45:30.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.585+0000 7fdbb1210900 20 add_watcher() i=7 2026-03-20T11:45:30.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.585+0000 7fdbb1210900 20 add_watcher() i=0 2026-03-20T11:45:30.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.585+0000 7fdbb1210900 20 add_watcher() i=3 2026-03-20T11:45:30.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.585+0000 7fdbb1210900 20 add_watcher() i=2 2026-03-20T11:45:30.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.585+0000 7fdbb1210900 2 all 8 watchers are set, enabling cache 2026-03-20T11:45:30.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.587+0000 7fdbb1210900 20 rgw_check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2026-03-20T11:45:30.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.587+0000 7fdbb1210900 20 rgw_check_secure_mon_conn(): mode 1 is insecure 2026-03-20T11:45:30.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.587+0000 7fdbb1210900 5 note: GC not initialized 2026-03-20T11:45:30.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.587+0000 7fdb5afdd640 20 reqs_thread_entry: start 2026-03-20T11:45:30.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.636+0000 7fdbb1210900 20 init_complete bucket index max shards: 11 2026-03-20T11:45:30.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.636+0000 7fdbb1210900 20 Filter name: none 2026-03-20T11:45:30.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.636+0000 7fdb58fd9640 20 reqs_thread_entry: start 2026-03-20T11:45:30.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.636+0000 7fdb0affd640 20 reqs_thread_entry: start 2026-03-20T11:45:30.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.638+0000 7fdbb1210900 10 rgw bucket trim manager: Watching bilog.trim 2026-03-20T11:45:30.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.638+0000 7fdbb1210900 20 cr:s=0x56024b863bb0:op=0x56024b8634b0:12BucketTrimCR: operate() 2026-03-20T11:45:30.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.638+0000 7fdbb1210900 4 collected 0 buckets for trim 2026-03-20T11:45:30.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.638+0000 7fdbb1210900 20 cr:s=0x56024b863bb0:op=0x56024b864f50:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:30.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.638+0000 7fdbb1210900 20 cr:s=0x56024b863bb0:op=0x56024b8634b0:12BucketTrimCR: operate() 2026-03-20T11:45:30.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.638+0000 7fdbb1210900 20 cr:s=0x56024b863bb0:op=0x56024b866730:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:30.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.638+0000 7fdbb1210900 20 cr:s=0x56024b863bb0:op=0x56024b866730:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:30.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.638+0000 7fdbb1210900 20 run: stack=0x56024b863bb0 is io blocked 2026-03-20T11:45:30.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.638+0000 7fdb837fe640 20 trim: bucket trim completed 2026-03-20T11:45:30.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.639+0000 7fdbb1210900 20 cr:s=0x56024b863bb0:op=0x56024b866730:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:30.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.639+0000 7fdbb1210900 20 cr:s=0x56024b863bb0:op=0x56024b866730:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:30.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.639+0000 7fdbb1210900 20 cr:s=0x56024b863bb0:op=0x56024b866730:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:30.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.639+0000 7fdbb1210900 20 cr:s=0x56024b863bb0:op=0x56024b866730:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:30.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.639+0000 7fdbb1210900 20 cr:s=0x56024b863bb0:op=0x56024b8634b0:12BucketTrimCR: operate() 2026-03-20T11:45:30.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.639+0000 7fdbb1210900 4 bucket index log processing completed in 0.000908441s 2026-03-20T11:45:30.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.639+0000 7fdbb1210900 15 stack 0x56024b863bb0 end 2026-03-20T11:45:30.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.639+0000 7fdbb1210900 20 run: stack=0x56024b863bb0 is done 2026-03-20T11:45:30.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.644+0000 7fdbb1210900 20 remove_watcher() i=5 2026-03-20T11:45:30.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.644+0000 7fdbb1210900 2 removed watcher, disabling cache 2026-03-20T11:45:30.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.644+0000 7fdbb1210900 20 remove_watcher() i=3 2026-03-20T11:45:30.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.644+0000 7fdbb1210900 20 remove_watcher() i=6 2026-03-20T11:45:30.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.644+0000 7fdbb1210900 20 remove_watcher() i=4 2026-03-20T11:45:30.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.645+0000 7fdbb1210900 20 remove_watcher() i=1 2026-03-20T11:45:30.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.645+0000 7fdbb1210900 20 remove_watcher() i=0 2026-03-20T11:45:30.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.645+0000 7fdbb1210900 20 remove_watcher() i=7 2026-03-20T11:45:30.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.645+0000 7fdbb1210900 20 remove_watcher() i=2 2026-03-20T11:45:30.648 INFO:rgw_multi.tests:create bucket zone=a1 name=dipepu-59 2026-03-20T11:45:30.686 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:30.686 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:30.686 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:45:30.724 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:30.724 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:30.798 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:45:30.809 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774006018.833900_51.1","last_update":"2026-03-20T11:26:58.833900Z"},{"marker":"1_1774006026.820020_59.1","last_update":"2026-03-20T11:27:06.820020Z"},{"marker":"1_1774007130.662708_255.1","last_update":"2026-03-20T11:45:30.662708Z"},{"marker":"1_1774007039.336409_210.1","last_update":"2026-03-20T11:43:59.336409Z"}] 2026-03-20T11:45:30.809 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:45:30.809 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:30.809 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:30.810 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:45:30.886 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:30.886 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:30.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.969+0000 7faad5fce900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:45:30.968 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:30.971+0000 7faad5fce900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:45:30.979 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774006018.833900_51.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:26:58.833900Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774006026.820020_59.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:27:06.820020Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774007130.662708_255.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:30.662708Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774007039.336409_210.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:59.336409Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:45:30Z"} 2026-03-20T11:45:30.979 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:45:30.979 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-59', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:45:30.979 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-59', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:45:30.979 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-59 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:45:31.063 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:31.063 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:31.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:31.159+0000 7f959d7cc900 1 waiting to reach incremental sync.. 2026-03-20T11:45:36.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.160+0000 7f959d7cc900 1 bucket sync caught up with source: 2026-03-20T11:45:36.158 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, 00000000001.44.6, , , , 00000000001.24.6, , , 00000000001.44.6, , 00000000001.39.6] 2026-03-20T11:45:36.158 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, 00000000001.44.6, , , , 00000000001.24.6, , , 00000000001.44.6, , 00000000001.39.6] 2026-03-20T11:45:36.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.160+0000 7f959d7cc900 0 bucket checkpoint complete 2026-03-20T11:45:36.169 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-59', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:45:36.169 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-59', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:45:36.169 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-59 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:45:36.211 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:36.211 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:36.287 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.289+0000 7f7298152900 1 bucket sync caught up with source: 2026-03-20T11:45:36.287 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, 00000000001.65.6, , , , 00000000001.44.6, , , 00000000001.58.6, , 00000000001.63.6] 2026-03-20T11:45:36.287 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, 00000000001.65.6, , , , 00000000001.44.6, , , 00000000001.58.6, , 00000000001.63.6] 2026-03-20T11:45:36.287 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.289+0000 7f7298152900 0 bucket checkpoint complete 2026-03-20T11:45:36.295 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-59 zones={a2, a1} 2026-03-20T11:45:36.302 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:45:36.302 DEBUG:tasks.rgw_multi.zone_rados:o=a, v=null 2026-03-20T11:45:36.302 DEBUG:tasks.rgw_multi.zone_rados:o=b, v=null 2026-03-20T11:45:36.302 DEBUG:tasks.rgw_multi.zone_rados:o=c, v=null 2026-03-20T11:45:36.302 DEBUG:tasks.rgw_multi.zone_rados:o=d, v=null 2026-03-20T11:45:36.305 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:45:36.305 DEBUG:tasks.rgw_multi.zone_rados:o=a, v=null 2026-03-20T11:45:36.305 DEBUG:tasks.rgw_multi.zone_rados:o=b, v=null 2026-03-20T11:45:36.305 DEBUG:tasks.rgw_multi.zone_rados:o=c, v=null 2026-03-20T11:45:36.305 DEBUG:tasks.rgw_multi.zone_rados:o=d, v=null 2026-03-20T11:45:36.305 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2026-03-20T11:45:36.361 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2026-03-20T11:45:36.418 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2026-03-20T11:45:36.470 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2026-03-20T11:45:36.526 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-59 zones={a2, a1} 2026-03-20T11:45:36.526 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '1', '--rgw-sync-log-trim-min-cold-buckets', '0', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:45:36.526 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '1', '--rgw-sync-log-trim-min-cold-buckets', '0', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:45:36.526 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bilog autotrim --rgw-sync-log-trim-max-buckets 1 --rgw-sync-log-trim-min-cold-buckets 0 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 20 --debug-ms 0 2026-03-20T11:45:36.562 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:36.562 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:36.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.578+0000 7fcae5810900 20 rados->read ofs=0 len=0 2026-03-20T11:45:36.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.579+0000 7fcae5810900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:45:36.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.579+0000 7fcae5810900 20 rados->read ofs=0 len=0 2026-03-20T11:45:36.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.579+0000 7fcae5810900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:45:36.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.579+0000 7fcae5810900 20 realm test-realm 6a789b89-c2b8-45f5-82cc-fb80234d0c6c 2026-03-20T11:45:36.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.579+0000 7fcae5810900 20 rados->read ofs=0 len=0 2026-03-20T11:45:36.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.579+0000 7fcae5810900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:45:36.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.579+0000 7fcae5810900 20 rados->read ofs=0 len=0 2026-03-20T11:45:36.578 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.580+0000 7fcae5810900 20 rados_obj.operate() r=0 bl.length=10 2026-03-20T11:45:36.578 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.580+0000 7fcae5810900 20 rados->read ofs=0 len=0 2026-03-20T11:45:36.578 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.580+0000 7fcae5810900 20 rados_obj.operate() r=0 bl.length=2020 2026-03-20T11:45:36.578 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.580+0000 7fcae5810900 20 rados->read ofs=0 len=0 2026-03-20T11:45:36.578 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.581+0000 7fcae5810900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:45:36.578 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.581+0000 7fcae5810900 20 rados->read ofs=0 len=0 2026-03-20T11:45:36.579 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.581+0000 7fcae5810900 20 rados_obj.operate() r=0 bl.length=1030 2026-03-20T11:45:36.579 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.581+0000 7fcae5810900 20 zone a1 found 2026-03-20T11:45:36.579 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.581+0000 7fcae5810900 4 Realm: test-realm (6a789b89-c2b8-45f5-82cc-fb80234d0c6c) 2026-03-20T11:45:36.579 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.581+0000 7fcae5810900 4 ZoneGroup: a (a) 2026-03-20T11:45:36.579 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.581+0000 7fcae5810900 4 Zone: a1 (4b634ebf-58f5-45bc-839a-cb4b425e773a) 2026-03-20T11:45:36.579 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.581+0000 7fcae5810900 4 using period configuration: cb2fcdce-2c1b-429a-8601-a284ee3a2381:12 2026-03-20T11:45:36.579 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.581+0000 7fcae5810900 20 period zonegroup name a 2026-03-20T11:45:36.579 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.581+0000 7fcae5810900 20 using current period zonegroup a 2026-03-20T11:45:36.579 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.581+0000 7fcae5810900 20 rados->read ofs=0 len=0 2026-03-20T11:45:36.579 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.582+0000 7fcae5810900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:36.579 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.582+0000 7fcae5810900 20 rados->read ofs=0 len=0 2026-03-20T11:45:36.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.582+0000 7fcae5810900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:36.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.582+0000 7fcae5810900 20 init(): pipe_map (zone=4b634ebf-58f5-45bc-839a-cb4b425e773a bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:45:36.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.582+0000 7fcae5810900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:36.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.582+0000 7fcae5810900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:36.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.582+0000 7fcae5810900 20 rados->read ofs=0 len=0 2026-03-20T11:45:36.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.582+0000 7fcae5810900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:36.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.582+0000 7fcae5810900 20 rados->read ofs=0 len=0 2026-03-20T11:45:36.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.582+0000 7fcae5810900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:36.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.582+0000 7fcae5810900 20 init(): pipe_map (zone=51055576-6662-409d-8abb-ecaeefd3dd80 bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:45:36.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.582+0000 7fcae5810900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:36.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.582+0000 7fcae5810900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:36.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.582+0000 7fcae5810900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:36.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.582+0000 7fcae5810900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:36.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.582+0000 7fcae5810900 20 started sync module instance, tier type = 2026-03-20T11:45:36.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.582+0000 7fcae5810900 20 generating connection object for zone a2 id 51055576-6662-409d-8abb-ecaeefd3dd80 2026-03-20T11:45:36.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.582+0000 7fcae5810900 20 started zone id=4b634ebf-58f5-45bc-839a-cb4b425e773a (name=a1) with tier type = 2026-03-20T11:45:36.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.585+0000 7fcae5810900 20 add_watcher() i=0 2026-03-20T11:45:36.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.586+0000 7fcae5810900 20 add_watcher() i=1 2026-03-20T11:45:36.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.587+0000 7fcae5810900 20 add_watcher() i=2 2026-03-20T11:45:36.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.587+0000 7fcae5810900 20 add_watcher() i=4 2026-03-20T11:45:36.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.587+0000 7fcae5810900 20 add_watcher() i=7 2026-03-20T11:45:36.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.587+0000 7fcae5810900 20 add_watcher() i=5 2026-03-20T11:45:36.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.587+0000 7fcae5810900 20 add_watcher() i=6 2026-03-20T11:45:36.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.588+0000 7fcae5810900 20 add_watcher() i=3 2026-03-20T11:45:36.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.588+0000 7fcae5810900 2 all 8 watchers are set, enabling cache 2026-03-20T11:45:36.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.590+0000 7fcae5810900 20 rgw_check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2026-03-20T11:45:36.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.590+0000 7fcae5810900 20 rgw_check_secure_mon_conn(): mode 1 is insecure 2026-03-20T11:45:36.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.590+0000 7fcae5810900 5 note: GC not initialized 2026-03-20T11:45:36.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.590+0000 7fca8e7e4640 20 reqs_thread_entry: start 2026-03-20T11:45:36.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.635+0000 7fcae5810900 20 init_complete bucket index max shards: 11 2026-03-20T11:45:36.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.635+0000 7fcae5810900 20 Filter name: none 2026-03-20T11:45:36.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.635+0000 7fca3ffff640 20 reqs_thread_entry: start 2026-03-20T11:45:36.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.635+0000 7fca3e7fc640 20 reqs_thread_entry: start 2026-03-20T11:45:36.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.636+0000 7fcae5810900 10 rgw bucket trim manager: Watching bilog.trim 2026-03-20T11:45:36.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.636+0000 7fcae5810900 20 cr:s=0x5637ebba0c10:op=0x5637ebba0e50:12BucketTrimCR: operate() 2026-03-20T11:45:36.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.636+0000 7fcae5810900 10 fetching active bucket counters 2026-03-20T11:45:36.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.636+0000 7fcae5810900 20 cr:s=0x5637ebba0c10:op=0x5637ebba3570:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:36.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.636+0000 7fcae5810900 20 cr:s=0x5637ebba0c10:op=0x5637ebba3570:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:36.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.636+0000 7fcae5810900 20 run: stack=0x5637ebba0c10 is io blocked 2026-03-20T11:45:36.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.636+0000 7fcaaffff640 20 trim: get_bucket_counters: [] 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcae5810900 20 cr:s=0x5637ebba0c10:op=0x5637ebba3570:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcae5810900 20 cr:s=0x5637ebba0c10:op=0x5637ebba3570:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcae5810900 20 cr:s=0x5637ebba0c10:op=0x5637ebba3570:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcae5810900 20 cr:s=0x5637ebba0c10:op=0x5637ebba3570:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcae5810900 20 cr:s=0x5637ebba0c10:op=0x5637ebba0e50:12BucketTrimCR: operate() 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcae5810900 4 collected 1 buckets for trim 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcae5810900 20 cr:s=0x5637ebba0c10:op=0x5637ebba4370:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcae5810900 20 run: stack=0x5637ebba0c10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcae5810900 20 cr:s=0x5637ebba4050:op=0x5637ebba2560:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcae5810900 4 starting trim on bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcae5810900 20 cr:s=0x5637ebba4050:op=0x5637ebba4950:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcae5810900 20 cr:s=0x5637ebba4050:op=0x5637ebba4950:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcae5810900 20 enqueued request req=0x5637ebba3830 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcae5810900 20 RGWWQ: 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcae5810900 20 req: 0x5637ebba3830 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcae5810900 20 run: stack=0x5637ebba4050 is io blocked 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcaa3fff640 20 dequeued request req=0x5637ebba3830 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcaa3fff640 20 RGWWQ: empty 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcaa3fff640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 : miss 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcaa3fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcaa3fff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcaa3fff640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 info.flags=0x17 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcaa3fff640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 to cache LRU end 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcaa3fff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcaa3fff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcaa3fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcaa3fff640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-59 : miss 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcaa3fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcaa3fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcaa3fff640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-59 info.flags=0x0 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcaa3fff640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-59 to cache LRU end 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcaa3fff640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-59 : miss 2026-03-20T11:45:36.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.637+0000 7fcaa3fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:36.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcaa3fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:36.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcaa3fff640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-59 info.flags=0x0 2026-03-20T11:45:36.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcaa3fff640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-59 to cache LRU end 2026-03-20T11:45:36.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcaa3fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19): adding source pipe: {s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:36.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcaa3fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19): adding dest pipe: {s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:36.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcaa3fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:36.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcaa3fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:36.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcaa3fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:45:36.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 20 cr:s=0x5637ebba4050:op=0x5637ebba4950:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:36.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 20 cr:s=0x5637ebba4050:op=0x5637ebba4950:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:36.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 20 cr:s=0x5637ebba4050:op=0x5637ebba4950:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:36.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 20 cr:s=0x5637ebba4050:op=0x5637ebba4950:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:36.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 20 cr:s=0x5637ebba4050:op=0x5637ebba2560:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:36.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 20 cr:s=0x5637ebba3ac0:op=0x5637ebba4950:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:36.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 20 cr:s=0x5637ebba4050:op=0x5637ebba2560:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:36.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 20 run: stack=0x5637ebba4050 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:36.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 20 cr:s=0x5637ebba3ac0:op=0x5637ebba5f60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:36.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 20 cr:s=0x5637ebba3ac0:op=0x5637ebba5f60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:36.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:36.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:36 +0000 2026-03-20T11:45:36.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:36.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:36 +0000 2026-03-20T11:45:36.636 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:36.636 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114536Z 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:36 +0000 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114536Z 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = dea41ddf2702d950c9eda0650c01be58b33df84ed29de8a75e351427c5a7c6ec 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114536Z 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr:dea41ddf2702d950c9eda0650c01be58b33df84ed29de8a75e351427c5a7c6ec 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 18e23ca26090ade16ba2c69a73828ec9d3834e2224fcb0a26ae80540cbfcc04e 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=18e23ca26090ade16ba2c69a73828ec9d3834e2224fcb0a26ae80540cbfcc04e 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114536Z 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 20 register_request mgr=0x7ffd5e724cf0 req_data->id=0, curl_handle=0x5637ebba54a0 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fcae5810900 20 run: stack=0x5637ebba3ac0 is io blocked 2026-03-20T11:45:36.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.638+0000 7fca3e7fc640 20 link_request req_data=0x5637ebba85f0 req_data->id=0, curl_handle=0x5637ebba54a0 2026-03-20T11:45:36.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.640+0000 7fcae5810900 20 cr:s=0x5637ebba3ac0:op=0x5637ebba5f60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 20 cr:s=0x5637ebba3ac0:op=0x5637ebba5f60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 20 cr:s=0x5637ebba3ac0:op=0x5637ebba5f60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 20 cr:s=0x5637ebba3ac0:op=0x5637ebba5f60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 20 cr:s=0x5637ebba3ac0:op=0x5637ebba4950:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 15 stack 0x5637ebba3ac0 end 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 20 run: stack=0x5637ebba3ac0 is done 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 20 cr:s=0x5637ebba4050:op=0x5637ebba2560:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 20 collect(): s=0x5637ebba4050 stack=0x5637ebba3ac0 is complete 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 10 min_generation is 0 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 10 trimming bilogs for bucket=:dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) markers=[,00000000001.44.6,,,,00000000001.24.6,,,00000000001.44.6,,00000000001.39.6], shards=11 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 20 cr:s=0x5637ebba4050:op=0x5637ebba4950:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 10 trimming bilog shard 1 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.44.6 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 10 trimming bilog shard 5 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.24.6 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 10 trimming bilog shard 8 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.44.6 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 10 trimming bilog shard 10 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.39.6 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 20 run: stack=0x5637ebba4050 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 20 cr:s=0x5637ebba3ac0:op=0x5637ebbba6c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 20 cr:s=0x5637ebbb9a60:op=0x5637ebbe96e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 20 cr:s=0x5637ebbb94f0:op=0x5637ebbea530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 20 cr:s=0x5637ebbebf60:op=0x5637ebbbb4d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 20 cr:s=0x5637ebba3ac0:op=0x5637ebbba6c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.1 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 20 run: stack=0x5637ebba3ac0 is io blocked 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 20 cr:s=0x5637ebbb9a60:op=0x5637ebbe96e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.5 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 20 run: stack=0x5637ebbb9a60 is io blocked 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 20 cr:s=0x5637ebbb94f0:op=0x5637ebbea530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.8 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 20 run: stack=0x5637ebbb94f0 is io blocked 2026-03-20T11:45:36.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 20 cr:s=0x5637ebbebf60:op=0x5637ebbbb4d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:36.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.10 2026-03-20T11:45:36.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.641+0000 7fcae5810900 20 run: stack=0x5637ebbebf60 is io blocked 2026-03-20T11:45:36.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 cr:s=0x5637ebba3ac0:op=0x5637ebbba6c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:36.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 cr:s=0x5637ebba3ac0:op=0x5637ebbba6c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:36.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 cr:s=0x5637ebba3ac0:op=0x5637ebbba6c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:36.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 cr:s=0x5637ebba3ac0:op=0x5637ebbba6c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:36.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 15 stack 0x5637ebba3ac0 end 2026-03-20T11:45:36.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 run: stack=0x5637ebba3ac0 is done 2026-03-20T11:45:36.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 cr:s=0x5637ebba4050:op=0x5637ebba4950:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:36.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 run: stack=0x5637ebba4050 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:36.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 cr:s=0x5637ebbebf60:op=0x5637ebbbb4d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:36.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 cr:s=0x5637ebbebf60:op=0x5637ebbbb4d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:36.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 cr:s=0x5637ebbebf60:op=0x5637ebbbb4d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:36.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 cr:s=0x5637ebbebf60:op=0x5637ebbbb4d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:36.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 15 stack 0x5637ebbebf60 end 2026-03-20T11:45:36.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 run: stack=0x5637ebbebf60 is done 2026-03-20T11:45:36.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 cr:s=0x5637ebba4050:op=0x5637ebba4950:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:36.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 run: stack=0x5637ebba4050 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 cr:s=0x5637ebbb94f0:op=0x5637ebbea530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 cr:s=0x5637ebbb94f0:op=0x5637ebbea530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 cr:s=0x5637ebbb94f0:op=0x5637ebbea530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 cr:s=0x5637ebbb94f0:op=0x5637ebbea530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 15 stack 0x5637ebbb94f0 end 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 run: stack=0x5637ebbb94f0 is done 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 cr:s=0x5637ebba4050:op=0x5637ebba4950:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 run: stack=0x5637ebba4050 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 cr:s=0x5637ebbb9a60:op=0x5637ebbe96e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 cr:s=0x5637ebbb9a60:op=0x5637ebbe96e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 cr:s=0x5637ebbb9a60:op=0x5637ebbe96e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 cr:s=0x5637ebbb9a60:op=0x5637ebbe96e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 15 stack 0x5637ebbb9a60 end 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 run: stack=0x5637ebbb9a60 is done 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 cr:s=0x5637ebba4050:op=0x5637ebba4950:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 cr:s=0x5637ebba4050:op=0x5637ebba2560:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 trim: trimmed bucket instance dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 15 stack 0x5637ebba4050 end 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 run: stack=0x5637ebba4050 is done 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 cr:s=0x5637ebba0c10:op=0x5637ebba4370:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 cr:s=0x5637ebba0c10:op=0x5637ebba0e50:12BucketTrimCR: operate() 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 cr:s=0x5637ebba0c10:op=0x5637ebbaf290:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 cr:s=0x5637ebba0c10:op=0x5637ebbaf290:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.642+0000 7fcae5810900 20 run: stack=0x5637ebba0c10 is io blocked 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.643+0000 7fcaaf7fe640 20 trim: bucket trim completed 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.643+0000 7fcae5810900 20 cr:s=0x5637ebba0c10:op=0x5637ebbaf290:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.643+0000 7fcae5810900 20 cr:s=0x5637ebba0c10:op=0x5637ebbaf290:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.643+0000 7fcae5810900 20 cr:s=0x5637ebba0c10:op=0x5637ebbaf290:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.643+0000 7fcae5810900 20 cr:s=0x5637ebba0c10:op=0x5637ebbaf290:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.643+0000 7fcae5810900 20 cr:s=0x5637ebba0c10:op=0x5637ebba0e50:12BucketTrimCR: operate() 2026-03-20T11:45:36.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.643+0000 7fcae5810900 4 bucket index log processing completed in 0.007014329s 2026-03-20T11:45:36.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.643+0000 7fcae5810900 15 stack 0x5637ebba0c10 end 2026-03-20T11:45:36.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.643+0000 7fcae5810900 20 run: stack=0x5637ebba0c10 is done 2026-03-20T11:45:36.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.648+0000 7fcae5810900 20 remove_watcher() i=1 2026-03-20T11:45:36.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.648+0000 7fcae5810900 2 removed watcher, disabling cache 2026-03-20T11:45:36.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.648+0000 7fcae5810900 20 remove_watcher() i=0 2026-03-20T11:45:36.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.648+0000 7fcae5810900 20 remove_watcher() i=2 2026-03-20T11:45:36.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.648+0000 7fcae5810900 20 remove_watcher() i=7 2026-03-20T11:45:36.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.648+0000 7fcae5810900 20 remove_watcher() i=4 2026-03-20T11:45:36.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.648+0000 7fcae5810900 20 remove_watcher() i=5 2026-03-20T11:45:36.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.648+0000 7fcae5810900 20 remove_watcher() i=3 2026-03-20T11:45:36.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:36.648+0000 7fcae5810900 20 remove_watcher() i=6 2026-03-20T11:45:36.650 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bilog', 'list', '--bucket', 'dipepu-59', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:36.651 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bilog', 'list', '--bucket', 'dipepu-59', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:36.651 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bilog list --bucket dipepu-59 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:45:36.727 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:36.727 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:36.813 INFO:teuthology.orchestra.run.vm02.stdout:[] 2026-03-20T11:45:36.813 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bilog', 'list', '--bucket', 'dipepu-58', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:36.813 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bilog', 'list', '--bucket', 'dipepu-58', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:36.813 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bilog list --bucket dipepu-58 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:45:36.889 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:36.889 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:36.978 INFO:teuthology.orchestra.run.vm02.stdout:[{"op_id":"00000000001.77.6","op_tag":"4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5105513999792218550","op":"write","object":"a","instance":"","state":"complete","index_ver":1,"timestamp":"2026-03-20T11:45:18.931676648Z","ver":{"pool":9,"epoch":18},"bilog_flags":0,"versioned":false,"owner":"","owner_display_name":"","zones_trace":[{"entry":"4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18"}]},{"op_id":"00000000001.65.6","op_tag":"4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7846608834922138223","op":"write","object":"d","instance":"","state":"complete","index_ver":1,"timestamp":"2026-03-20T11:45:18.958133761Z","ver":{"pool":9,"epoch":7},"bilog_flags":0,"versioned":false,"owner":"","owner_display_name":"","zones_trace":[{"entry":"4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18"}]},{"op_id":"00000000001.20.6","op_tag":"4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.849619897125843729","op":"write","object":"c","instance":"","state":"complete","index_ver":1,"timestamp":"2026-03-20T11:45:18.950130250Z","ver":{"pool":9,"epoch":6},"bilog_flags":0,"versioned":false,"owner":"","owner_display_name":"","zones_trace":[{"entry":"4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18"}]},{"op_id":"00000000001.28.6","op_tag":"4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2949062215011903421","op":"write","object":"b","instance":"","state":"complete","index_ver":1,"timestamp":"2026-03-20T11:45:18.941930783Z","ver":{"pool":9,"epoch":7},"bilog_flags":0,"versioned":false,"owner":"","owner_display_name":"","zones_trace":[{"entry":"4b634ebf-58f5-45bc-839a-cb4b425e773a:dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18"}]}] 2026-03-20T11:45:36.978 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '999', '--rgw-sync-log-trim-min-cold-buckets', '999', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:45:36.978 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '999', '--rgw-sync-log-trim-min-cold-buckets', '999', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:45:36.978 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bilog autotrim --rgw-sync-log-trim-max-buckets 999 --rgw-sync-log-trim-min-cold-buckets 999 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 20 --debug-ms 0 2026-03-20T11:45:37.055 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:37.055 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:37.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.074+0000 7f8f59593900 20 rados->read ofs=0 len=0 2026-03-20T11:45:37.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.075+0000 7f8f59593900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:45:37.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.075+0000 7f8f59593900 20 rados->read ofs=0 len=0 2026-03-20T11:45:37.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.076+0000 7f8f59593900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:45:37.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.076+0000 7f8f59593900 20 realm test-realm 6a789b89-c2b8-45f5-82cc-fb80234d0c6c 2026-03-20T11:45:37.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.076+0000 7f8f59593900 20 rados->read ofs=0 len=0 2026-03-20T11:45:37.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.076+0000 7f8f59593900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:45:37.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.076+0000 7f8f59593900 20 rados->read ofs=0 len=0 2026-03-20T11:45:37.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.077+0000 7f8f59593900 20 rados_obj.operate() r=0 bl.length=10 2026-03-20T11:45:37.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.077+0000 7f8f59593900 20 rados->read ofs=0 len=0 2026-03-20T11:45:37.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.077+0000 7f8f59593900 20 rados_obj.operate() r=0 bl.length=2020 2026-03-20T11:45:37.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.077+0000 7f8f59593900 20 rados->read ofs=0 len=0 2026-03-20T11:45:37.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.077+0000 7f8f59593900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:45:37.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.077+0000 7f8f59593900 20 rados->read ofs=0 len=0 2026-03-20T11:45:37.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.078+0000 7f8f59593900 20 rados_obj.operate() r=0 bl.length=1030 2026-03-20T11:45:37.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.078+0000 7f8f59593900 20 zone a1 found 2026-03-20T11:45:37.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.078+0000 7f8f59593900 4 Realm: test-realm (6a789b89-c2b8-45f5-82cc-fb80234d0c6c) 2026-03-20T11:45:37.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.078+0000 7f8f59593900 4 ZoneGroup: a (a) 2026-03-20T11:45:37.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.078+0000 7f8f59593900 4 Zone: a1 (4b634ebf-58f5-45bc-839a-cb4b425e773a) 2026-03-20T11:45:37.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.078+0000 7f8f59593900 4 using period configuration: cb2fcdce-2c1b-429a-8601-a284ee3a2381:12 2026-03-20T11:45:37.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.078+0000 7f8f59593900 20 period zonegroup name a 2026-03-20T11:45:37.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.078+0000 7f8f59593900 20 using current period zonegroup a 2026-03-20T11:45:37.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.078+0000 7f8f59593900 20 rados->read ofs=0 len=0 2026-03-20T11:45:37.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.078+0000 7f8f59593900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.078+0000 7f8f59593900 20 rados->read ofs=0 len=0 2026-03-20T11:45:37.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.078+0000 7f8f59593900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.078+0000 7f8f59593900 20 init(): pipe_map (zone=4b634ebf-58f5-45bc-839a-cb4b425e773a bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:45:37.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.078+0000 7f8f59593900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.078+0000 7f8f59593900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.078+0000 7f8f59593900 20 rados->read ofs=0 len=0 2026-03-20T11:45:37.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.079+0000 7f8f59593900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.079+0000 7f8f59593900 20 rados->read ofs=0 len=0 2026-03-20T11:45:37.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.079+0000 7f8f59593900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.079+0000 7f8f59593900 20 init(): pipe_map (zone=51055576-6662-409d-8abb-ecaeefd3dd80 bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:45:37.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.079+0000 7f8f59593900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.079+0000 7f8f59593900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.079+0000 7f8f59593900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.079+0000 7f8f59593900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.079+0000 7f8f59593900 20 started sync module instance, tier type = 2026-03-20T11:45:37.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.079+0000 7f8f59593900 20 generating connection object for zone a2 id 51055576-6662-409d-8abb-ecaeefd3dd80 2026-03-20T11:45:37.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.079+0000 7f8f59593900 20 started zone id=4b634ebf-58f5-45bc-839a-cb4b425e773a (name=a1) with tier type = 2026-03-20T11:45:37.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.082+0000 7f8f59593900 20 add_watcher() i=5 2026-03-20T11:45:37.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.083+0000 7f8f59593900 20 add_watcher() i=1 2026-03-20T11:45:37.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.083+0000 7f8f59593900 20 add_watcher() i=0 2026-03-20T11:45:37.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.083+0000 7f8f59593900 20 add_watcher() i=2 2026-03-20T11:45:37.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.083+0000 7f8f59593900 20 add_watcher() i=7 2026-03-20T11:45:37.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.083+0000 7f8f59593900 20 add_watcher() i=4 2026-03-20T11:45:37.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.083+0000 7f8f59593900 20 add_watcher() i=6 2026-03-20T11:45:37.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.084+0000 7f8f59593900 20 add_watcher() i=3 2026-03-20T11:45:37.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.084+0000 7f8f59593900 2 all 8 watchers are set, enabling cache 2026-03-20T11:45:37.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.086+0000 7f8f59593900 20 rgw_check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2026-03-20T11:45:37.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.086+0000 7f8f59593900 20 rgw_check_secure_mon_conn(): mode 1 is insecure 2026-03-20T11:45:37.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.086+0000 7f8f59593900 5 note: GC not initialized 2026-03-20T11:45:37.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.086+0000 7f8f02fe5640 20 reqs_thread_entry: start 2026-03-20T11:45:37.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.133+0000 7f8f59593900 20 init_complete bucket index max shards: 11 2026-03-20T11:45:37.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.133+0000 7f8f59593900 20 Filter name: none 2026-03-20T11:45:37.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.134+0000 7f8f00fe1640 20 reqs_thread_entry: start 2026-03-20T11:45:37.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.134+0000 7f8eb2ffd640 20 reqs_thread_entry: start 2026-03-20T11:45:37.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.135+0000 7f8f59593900 10 rgw bucket trim manager: Watching bilog.trim 2026-03-20T11:45:37.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.135+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e1966d110:12BucketTrimCR: operate() 2026-03-20T11:45:37.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.135+0000 7f8f59593900 10 fetching active bucket counters 2026-03-20T11:45:37.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.135+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e1966f830:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:37.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.135+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e1966f830:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:37.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.135+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is io blocked 2026-03-20T11:45:37.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.136+0000 7f8f23fff640 20 trim: get_bucket_counters: [] 2026-03-20T11:45:37.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.136+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e1966f830:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:37.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.136+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e1966f830:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:37.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.136+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e1966f830:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:37.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.136+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e1966f830:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:37.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.136+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e1966d110:12BucketTrimCR: operate() 2026-03-20T11:45:37.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.136+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:37.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.136+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:37.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.136+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is io blocked 2026-03-20T11:45:37.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.136+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:37.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.136+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:37.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.136+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:37.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.136+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:37.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.136+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e1966d110:12BucketTrimCR: operate() 2026-03-20T11:45:37.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.136+0000 7f8f59593900 10 listing cold buckets from marker= 2026-03-20T11:45:37.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.136+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:14MetadataListCR: operate() 2026-03-20T11:45:37.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.136+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:14MetadataListCR: operate() 2026-03-20T11:45:37.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.136+0000 7f8f59593900 20 enqueued request req=0x7f8ee8001790 2026-03-20T11:45:37.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.136+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.136+0000 7f8f59593900 20 req: 0x7f8ee8001790 2026-03-20T11:45:37.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.136+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is io blocked 2026-03-20T11:45:37.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.136+0000 7f8f047e8640 20 dequeued request req=0x7f8ee8001790 2026-03-20T11:45:37.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.136+0000 7f8f047e8640 20 RGWWQ: empty 2026-03-20T11:45:37.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.136+0000 7f8f047e8640 20 rgw async rados processor: starting metadata listing at 2026-03-20T11:45:37.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.143+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-10 2026-03-20T11:45:37.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.143+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:45:37.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.150+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:45:37.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.156+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:45:37.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.162+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:45:37.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.167+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:45:37.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.173+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:45:37.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.178+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-19 2026-03-20T11:45:37.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.178+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-3 2026-03-20T11:45:37.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.178+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:45:37.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.184+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:45:37.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.189+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-17 2026-03-20T11:45:37.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.189+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:45:37.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.194+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-38 2026-03-20T11:45:37.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.194+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:45:37.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.200+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-24 2026-03-20T11:45:37.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.200+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-36 2026-03-20T11:45:37.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.200+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:45:37.203 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.205+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:45:37.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.211+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:45:37.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.216+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-44 2026-03-20T11:45:37.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.216+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-35 2026-03-20T11:45:37.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.216+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:45:37.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.222+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-16 2026-03-20T11:45:37.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.222+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:45:37.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.227+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:45:37.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.231+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:45:37.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.236+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-55 2026-03-20T11:45:37.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.236+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-13 2026-03-20T11:45:37.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.236+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-48 2026-03-20T11:45:37.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.236+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-25 2026-03-20T11:45:37.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.236+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-51 2026-03-20T11:45:37.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.236+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:45:37.238 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.241+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:45:37.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.245+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-46 2026-03-20T11:45:37.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.245+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:45:37.247 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.249+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:45:37.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.253+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:45:37.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.258+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-26 2026-03-20T11:45:37.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.258+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-20 2026-03-20T11:45:37.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.258+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:45:37.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.262+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:45:37.264 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.266+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-1 2026-03-20T11:45:37.264 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.266+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:45:37.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.270+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:45:37.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.273+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:45:37.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.277+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-52 2026-03-20T11:45:37.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.277+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-32 2026-03-20T11:45:37.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.277+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-42 2026-03-20T11:45:37.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.277+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:45:37.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.281+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:45:37.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.284+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-7 2026-03-20T11:45:37.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.284+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-54 2026-03-20T11:45:37.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.284+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-49 2026-03-20T11:45:37.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.284+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-47 2026-03-20T11:45:37.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.284+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 2026-03-20T11:45:37.285 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.287+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:45:37.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.292+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-11 2026-03-20T11:45:37.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.292+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-12 2026-03-20T11:45:37.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.292+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-41 2026-03-20T11:45:37.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.292+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-23 2026-03-20T11:45:37.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.292+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-15 2026-03-20T11:45:37.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.292+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-45 2026-03-20T11:45:37.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.292+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:45:37.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.295+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:45:37.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.298+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-9 2026-03-20T11:45:37.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.298+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:45:37.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.301+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-29 2026-03-20T11:45:37.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.301+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:45:37.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.303+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-31 2026-03-20T11:45:37.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.303+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:45:37.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.305+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-33 2026-03-20T11:45:37.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.305+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:45:37.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.308+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-4 2026-03-20T11:45:37.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.308+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:45:37.308 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.310+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:45:37.310 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.312+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:45:37.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.314+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-43 2026-03-20T11:45:37.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.314+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-30 2026-03-20T11:45:37.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.314+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-18 2026-03-20T11:45:37.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.314+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-2 2026-03-20T11:45:37.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.314+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:45:37.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.316+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-8 2026-03-20T11:45:37.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.316+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:45:37.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.319+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:45:37.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.321+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-58 2026-03-20T11:45:37.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.321+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:45:37.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.322+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 2026-03-20T11:45:37.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.324+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:45:37.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.325+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-27 2026-03-20T11:45:37.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.325+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-28 2026-03-20T11:45:37.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.325+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-59 2026-03-20T11:45:37.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.325+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:45:37.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.327+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:45:37.326 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.328+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-40 2026-03-20T11:45:37.326 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.328+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-22 2026-03-20T11:45:37.326 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.328+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:45:37.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.329+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-56 2026-03-20T11:45:37.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.329+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 2026-03-20T11:45:37.328 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.330+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:45:37.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.332+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:45:37.330 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.333+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:45:37.331 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.334+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-53 2026-03-20T11:45:37.331 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.334+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:45:37.332 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.335+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:45:37.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.336+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:45:37.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.336+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:45:37.335 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.337+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:45:37.336 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-34 2026-03-20T11:45:37.336 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-21 2026-03-20T11:45:37.336 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-14 2026-03-20T11:45:37.336 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-50 2026-03-20T11:45:37.336 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-57 2026-03-20T11:45:37.336 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:45:37.336 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-39 2026-03-20T11:45:37.336 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:45:37.336 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f047e8640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-37 2026-03-20T11:45:37.336 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:14MetadataListCR: operate() 2026-03-20T11:45:37.336 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:14MetadataListCR: operate() 2026-03-20T11:45:37.336 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:14MetadataListCR: operate() 2026-03-20T11:45:37.336 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:14MetadataListCR: operate() 2026-03-20T11:45:37.336 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e1966d110:12BucketTrimCR: operate() 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 4 collected 60 buckets for trim 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 4 starting trim on bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 4 starting trim on bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 4 starting trim on bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 4 starting trim on bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19670c10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e1967d630:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967dd80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967e4d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19670c10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 enqueued request req=0x557e19671a40 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 req: 0x557e19671a40 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 run: stack=0x557e196ad300 is io blocked 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e1967d630:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 enqueued request req=0x557e1967ec20 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 req: 0x557e19671a40 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 req: 0x557e1967ec20 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is io blocked 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967dd80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 enqueued request req=0x557e1967f010 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 req: 0x557e19671a40 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 req: 0x557e1967ec20 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f59593900 20 req: 0x557e1967f010 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f16ffd640 20 dequeued request req=0x557e19671a40 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f16ffd640 20 RGWWQ: 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f16ffd640 20 req: 0x557e1967ec20 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f16ffd640 20 req: 0x557e1967f010 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f16ffd640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 : miss 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.338+0000 7f8f16ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f59593900 20 run: stack=0x557e1957ead0 is io blocked 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967e4d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f17fff640 20 dequeued request req=0x557e1967ec20 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f17fff640 20 RGWWQ: 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f17fff640 20 req: 0x557e1967f010 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f177fe640 20 dequeued request req=0x557e1967f010 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f177fe640 20 RGWWQ: empty 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f59593900 20 enqueued request req=0x557e1967f430 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f59593900 20 req: 0x557e1967f430 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f59593900 20 run: stack=0x557e196700a0 is io blocked 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f17fff640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 : miss 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f17fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f177fe640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 : miss 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f177fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f167fc640 20 dequeued request req=0x557e1967f430 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f167fc640 20 RGWWQ: empty 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f167fc640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 : miss 2026-03-20T11:45:37.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f167fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f16ffd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f16ffd640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 info.flags=0x17 2026-03-20T11:45:37.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f16ffd640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 to cache LRU end 2026-03-20T11:45:37.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f16ffd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f16ffd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f16ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:45:37.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f16ffd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-55 : miss 2026-03-20T11:45:37.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f16ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f177fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f177fe640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 info.flags=0x17 2026-03-20T11:45:37.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f177fe640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 to cache LRU end 2026-03-20T11:45:37.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f177fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f177fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:45:37.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f167fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f177fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:45:37.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f177fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-31 : miss 2026-03-20T11:45:37.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f177fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f167fc640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 info.flags=0x17 2026-03-20T11:45:37.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f167fc640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 to cache LRU end 2026-03-20T11:45:37.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f167fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f167fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f167fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f177fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f177fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-31 info.flags=0x0 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f177fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-31 to cache LRU end 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f167fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-39 : miss 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f177fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-31 : miss 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f177fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.339+0000 7f8f167fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f17fff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f167fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f17fff640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 info.flags=0x17 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f17fff640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 to cache LRU end 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f17fff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f17fff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f167fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-39 info.flags=0x0 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f167fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-39 to cache LRU end 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f167fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-39 : miss 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f167fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f17fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f17fff640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-28 : miss 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f17fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f16ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f16ffd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-55 info.flags=0x0 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f16ffd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-55 to cache LRU end 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f16ffd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-55 : miss 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f16ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f167fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f167fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-39 info.flags=0x0 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f167fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-39 to cache LRU end 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f167fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding source pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f167fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding dest pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f167fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f167fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f167fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967e4d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967e4d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967e4d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967e4d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 cr:s=0x557e1966fdc0:op=0x557e1967f850:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 run: stack=0x557e196700a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 cr:s=0x557e1966fdc0:op=0x557e1967e4d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 cr:s=0x557e1966fdc0:op=0x557e1967e4d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f177fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f177fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-31 info.flags=0x0 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f177fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-31 to cache LRU end 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f177fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding source pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f177fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding dest pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.340 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 746c7a344608f13b2b8041594c5072825d53860eeb69e005ad649799b362f60f 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:746c7a344608f13b2b8041594c5072825d53860eeb69e005ad649799b362f60f 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f177fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f177fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f177fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 1321908b3f042359b98624d13482ba4d13910ce921c77336b911f4e1744dc586 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=1321908b3f042359b98624d13482ba4d13910ce921c77336b911f4e1744dc586 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=0, curl_handle=0x557e19681800 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 run: stack=0x557e1966fdc0 is io blocked 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967dd80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967dd80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967dd80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967dd80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 cr:s=0x557e19684450:op=0x557e19683e30:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 run: stack=0x557e1957ead0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 cr:s=0x557e19684450:op=0x557e196845e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 cr:s=0x557e19684450:op=0x557e196845e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8eb2ffd640 20 link_request req_data=0x557e196823d0 req_data->id=0, curl_handle=0x557e19681800 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.340+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 85d6bfcb3a8921def56253af2f583a259bde109f1ca95309902c6a0d60e58da4 2026-03-20T11:45:37.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:85d6bfcb3a8921def56253af2f583a259bde109f1ca95309902c6a0d60e58da4 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 8e63e13c6cb65bb3c224ba239e314d0d91200001c67f40e989dc3884a4076fa3 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=8e63e13c6cb65bb3c224ba239e314d0d91200001c67f40e989dc3884a4076fa3 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=1, curl_handle=0x557e196861c0 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 run: stack=0x557e19684450 is io blocked 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f17fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f17fff640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-28 info.flags=0x0 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f17fff640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-28 to cache LRU end 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f17fff640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-28 : miss 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f17fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f16ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f16ffd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-55 info.flags=0x0 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f16ffd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-55 to cache LRU end 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f16ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding source pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f16ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding dest pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f16ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f16ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f16ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8eb2ffd640 20 link_request req_data=0x557e196868d0 req_data->id=1, curl_handle=0x557e196861c0 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19670c10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19670c10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19670c10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19670c10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e19688070:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e19670c10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e19670c10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 98255d155d0987b969386a6dcac832772273b76aa8dbe44e8315209f04ffdd66 2026-03-20T11:45:37.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:98255d155d0987b969386a6dcac832772273b76aa8dbe44e8315209f04ffdd66 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 8f7dd24d0cfe4298244f64c21875a187faf0de419c08a0010106abc75cddc641 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=8f7dd24d0cfe4298244f64c21875a187faf0de419c08a0010106abc75cddc641 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=2, curl_handle=0x557e196b2160 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 run: stack=0x557e19688770 is io blocked 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8eb2ffd640 20 link_request req_data=0x557e196b2870 req_data->id=2, curl_handle=0x557e196b2160 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f17fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f17fff640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-28 info.flags=0x0 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f17fff640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-28 to cache LRU end 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f17fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding source pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f17fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding dest pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f17fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f17fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f17fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e1967d630:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e1967d630:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e1967d630:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e1967d630:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e196b4290:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e196b48e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e196b48e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 0a5c92806feeb847ad2657e850f9e511fba93466b476e09598f31adc9d128433 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:0a5c92806feeb847ad2657e850f9e511fba93466b476e09598f31adc9d128433 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 7578faaa0d73faed1981bff8cb6685c62b5d9d7449de25b70d49f98cc1049318 2026-03-20T11:45:37.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=7578faaa0d73faed1981bff8cb6685c62b5d9d7449de25b70d49f98cc1049318 2026-03-20T11:45:37.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=3, curl_handle=0x557e196b6260 2026-03-20T11:45:37.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is io blocked 2026-03-20T11:45:37.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.341+0000 7f8eb2ffd640 20 link_request req_data=0x557e196b6970 req_data->id=3, curl_handle=0x557e196b6260 2026-03-20T11:45:37.344 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:37.343+0000 7fcff0a2f640 -1 req 821654381715002249 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:37.344 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:37.344+0000 7fcfdda09640 -1 req 16428402344513812966 0.003000004s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f59593900 20 cr:s=0x557e19684450:op=0x557e196845e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f59593900 20 cr:s=0x557e19684450:op=0x557e196845e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f59593900 20 cr:s=0x557e19684450:op=0x557e19683e30:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f59593900 15 stack 0x557e19684450 end 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f59593900 20 run: stack=0x557e19684450 is done 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f59593900 20 collect(): s=0x557e1957ead0 stack=0x557e19684450 is complete 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f59593900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f59593900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f59593900 15 stack 0x557e1957ead0 end 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f59593900 20 stack->operate() returned ret=-22 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f59593900 20 run: stack=0x557e1957ead0 is done 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f59593900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f59593900 4 starting trim on bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e19684590:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e19684590:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f59593900 20 enqueued request req=0x557e1967f010 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f59593900 20 req: 0x557e1967f010 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f59593900 20 run: stack=0x557e1957ead0 is io blocked 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f15ffb640 20 dequeued request req=0x557e1967f010 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f15ffb640 20 RGWWQ: empty 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f15ffb640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 : miss 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f15ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.344+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e19670c10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f15ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f15ffb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 info.flags=0x17 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f15ffb640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 to cache LRU end 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f15ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f15ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f15ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e19670c10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e19670c10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e19670c10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f15ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-8 : miss 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e19688070:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 15 stack 0x557e19688770 end 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 run: stack=0x557e19688770 is done 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 collect(): s=0x557e196ad300 stack=0x557e19688770 is complete 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f15ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-55[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15]) markers=[,,,,,00000000006.98.6,,,,,], shards=11 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19683e10:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 10 trimming bilog shard 5 of :dipepu-55[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15]) at marker 00000000006.98.6 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196b0d30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196b0d30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15.5 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 run: stack=0x557e19688770 is io blocked 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e196b48e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e196b48e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e196b4290:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 15 stack 0x557e1967dc20 end 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is done 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 collect(): s=0x557e1957ee80 stack=0x557e1967dc20 is complete 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 15 stack 0x557e1957ee80 end 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 stack->operate() returned ret=-22 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is done 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 4 starting trim on bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196cc670:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196cc670:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 enqueued request req=0x557e1967ec20 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 req: 0x557e1967ec20 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is io blocked 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f157fa640 20 dequeued request req=0x557e1967ec20 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f157fa640 20 RGWWQ: empty 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f157fa640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 : miss 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.345+0000 7f8f157fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f15ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f15ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-8 info.flags=0x0 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f15ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-8 to cache LRU end 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f15ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-8 : miss 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f15ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f157fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f157fa640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 info.flags=0x17 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f157fa640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 to cache LRU end 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f157fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f157fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f157fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f157fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-16 : miss 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f157fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f15ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f15ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-8 info.flags=0x0 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f15ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-8 to cache LRU end 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f15ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding source pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f15ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding dest pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f15ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f15ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f15ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e19684590:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e19684590:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e19684590:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e19684590:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e196cb0a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f59593900 20 run: stack=0x557e1957ead0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.346+0000 7f8f59593900 20 cr:s=0x557e1966fdc0:op=0x557e1967e4d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f157fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f157fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-16 info.flags=0x0 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f157fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-16 to cache LRU end 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f157fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-16 : miss 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f157fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e19688050:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e1966fdc0:op=0x557e1967e4d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196b0d30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e19688050:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = f5784ead9e5927ca4416acb27abd740ea146c2017e4e2a3ef5959fa70338a190 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:f5784ead9e5927ca4416acb27abd740ea146c2017e4e2a3ef5959fa70338a190 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 35f2b0b83fda34923d98138e6b62fd0cd285799bec894354adce1b3552ebd392 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=35f2b0b83fda34923d98138e6b62fd0cd285799bec894354adce1b3552ebd392 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=4, curl_handle=0x557e19681800 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is io blocked 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e1966fdc0:op=0x557e1967e4d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196b0d30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e1966fdc0:op=0x557e1967e4d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196b0d30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e1966fdc0:op=0x557e1967f850:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8eb2ffd640 20 link_request req_data=0x557e196823d0 req_data->id=4, curl_handle=0x557e19681800 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 15 stack 0x557e1966fdc0 end 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 run: stack=0x557e1966fdc0 is done 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196b0d30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 15 stack 0x557e19688770 end 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 run: stack=0x557e19688770 is done 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 collect(): s=0x557e196700a0 stack=0x557e1966fdc0 is complete 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-39[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44]) markers=[,,,,,,,,00000000001.10.6,,], shards=11 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19683e10:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967e4d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 10 trimming bilog shard 8 of :dipepu-39[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44]) at marker 00000000001.10.6 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 run: stack=0x557e196700a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 15 stack 0x557e196ad300 end 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 run: stack=0x557e196ad300 is done 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c6ad0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c6ad0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44.8 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 run: stack=0x557e19688770 is io blocked 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 4 starting trim on bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196b4f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196b4f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 enqueued request req=0x557e1967f430 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 req: 0x557e1967f430 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 run: stack=0x557e196ad300 is io blocked 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f157fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f157fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-16 info.flags=0x0 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f157fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-16 to cache LRU end 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f157fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding source pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f157fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding dest pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f157fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f157fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f14ff9640 20 dequeued request req=0x557e1967f430 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f14ff9640 20 RGWWQ: empty 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f157fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f14ff9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 : miss 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196cc670:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196cc670:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196cc670:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f14ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196cc670:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e1966fdc0:op=0x557e196bea00:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e1966fdc0:op=0x557e19683e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 cr:s=0x557e1966fdc0:op=0x557e19683e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 92a763b4b569a168dcab6618152f24723df8a0cc7346e9021d527a1a36d95429 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:92a763b4b569a168dcab6618152f24723df8a0cc7346e9021d527a1a36d95429 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 7b01320dfafc6a3548f95241ddd9858d15b547e251627d6c2014db321c61f628 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=7b01320dfafc6a3548f95241ddd9858d15b547e251627d6c2014db321c61f628 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=5, curl_handle=0x557e196b6260 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f59593900 20 run: stack=0x557e1966fdc0 is io blocked 2026-03-20T11:45:37.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8eb2ffd640 20 link_request req_data=0x557e196b6970 req_data->id=5, curl_handle=0x557e196b6260 2026-03-20T11:45:37.349 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:37.350+0000 7fd09036e640 -1 req 5289802777682923716 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:37.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f14ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=314 2026-03-20T11:45:37.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f14ff9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 info.flags=0x17 2026-03-20T11:45:37.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f14ff9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 to cache LRU end 2026-03-20T11:45:37.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f14ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f14ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.347+0000 7f8f14ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:45:37.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.348+0000 7f8f14ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-43 : miss 2026-03-20T11:45:37.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.348+0000 7f8f14ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.348+0000 7f8f14ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.348+0000 7f8f14ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-43 info.flags=0x0 2026-03-20T11:45:37.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.348+0000 7f8f14ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-43 to cache LRU end 2026-03-20T11:45:37.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.348+0000 7f8f14ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-43 : miss 2026-03-20T11:45:37.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.348+0000 7f8f14ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.348+0000 7f8f14ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.348+0000 7f8f14ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-43 info.flags=0x0 2026-03-20T11:45:37.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.348+0000 7f8f14ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-43 to cache LRU end 2026-03-20T11:45:37.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.348+0000 7f8f14ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding source pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f14ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding dest pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f14ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f14ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f14ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:45:37.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196b4f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196b4f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196b4f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196b4f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 cr:s=0x557e196b8e90:op=0x557e1967f7b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 cr:s=0x557e196b8e90:op=0x557e19670c10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 cr:s=0x557e196b8e90:op=0x557e19670c10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 384aa1d06d45b1d9d2ce8376e2d20444ddcfff7f56c274002c4739d6c39a52d8 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:384aa1d06d45b1d9d2ce8376e2d20444ddcfff7f56c274002c4739d6c39a52d8 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = e45c1ca05b5df88d31a56151d4cd01cdc65096511188e70c7b5bb6316d732b05 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=e45c1ca05b5df88d31a56151d4cd01cdc65096511188e70c7b5bb6316d732b05 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=6, curl_handle=0x557e196b2160 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 run: stack=0x557e196b8e90 is io blocked 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8eb2ffd640 20 link_request req_data=0x557e196b2870 req_data->id=6, curl_handle=0x557e196b2160 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c6ad0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c6ad0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c6ad0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c6ad0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 15 stack 0x557e19688770 end 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 run: stack=0x557e19688770 is done 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967e4d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 15 stack 0x557e196700a0 end 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 run: stack=0x557e196700a0 is done 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 4 starting trim on bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196b4f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196b4f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 enqueued request req=0x557e19670280 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 req: 0x557e19670280 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f59593900 20 run: stack=0x557e196700a0 is io blocked 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f0f7fe640 20 dequeued request req=0x557e19670280 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f0f7fe640 20 RGWWQ: empty 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f0f7fe640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 : miss 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.349+0000 7f8f0f7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f0f7fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f0f7fe640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 info.flags=0x17 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f0f7fe640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 to cache LRU end 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f0f7fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f0f7fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f0f7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f0f7fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-10 : miss 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f0f7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f0f7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f0f7fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-10 info.flags=0x0 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f0f7fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-10 to cache LRU end 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f0f7fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-10 : miss 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f0f7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f0f7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f0f7fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-10 info.flags=0x0 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f0f7fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-10 to cache LRU end 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f0f7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding source pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f0f7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding dest pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f0f7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f0f7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f0f7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196b4f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196b4f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196b4f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196b4f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196bfcc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 20 run: stack=0x557e196700a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196b4f10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196b4f10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.352 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.353 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = ac22b483f0fb0bb978d0fe7dbec3f9336d183374157abcd4a2cffaf7dfb7fbcd 2026-03-20T11:45:37.353 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.353 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.353 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.353 INFO:teuthology.orchestra.run.vm02.stderr:ac22b483f0fb0bb978d0fe7dbec3f9336d183374157abcd4a2cffaf7dfb7fbcd 2026-03-20T11:45:37.353 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.353 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.353 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.353 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.353 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = c1943ab15c436ddab0ed5a5fa6b583350b0b95fc6e79b37ed8222d3bd9ebbb4b 2026-03-20T11:45:37.353 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c1943ab15c436ddab0ed5a5fa6b583350b0b95fc6e79b37ed8222d3bd9ebbb4b 2026-03-20T11:45:37.353 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.353 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.353 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.353 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=7, curl_handle=0x557e196861c0 2026-03-20T11:45:37.353 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.350+0000 7f8f59593900 20 run: stack=0x557e19688770 is io blocked 2026-03-20T11:45:37.353 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.351+0000 7f8eb2ffd640 20 link_request req_data=0x557e196868d0 req_data->id=7, curl_handle=0x557e196861c0 2026-03-20T11:45:37.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e19688050:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e19688050:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e1966fdc0:op=0x557e19683e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e19688050:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e1966fdc0:op=0x557e19683e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e19688050:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e1966fdc0:op=0x557e19683e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e196cb0a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 15 stack 0x557e1967dc20 end 2026-03-20T11:45:37.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is done 2026-03-20T11:45:37.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e1966fdc0:op=0x557e19683e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 collect(): s=0x557e1957ead0 stack=0x557e1967dc20 is complete 2026-03-20T11:45:37.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) markers=[,00000000002.12.6,,00000000001.5.6,,00000000001.4.6,00000000002.12.6,00000000001.8.6,00000000001.14.6,00000000001.9.6,], shards=11 2026-03-20T11:45:37.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e1966fdc0:op=0x557e196bea00:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 15 stack 0x557e1966fdc0 end 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 run: stack=0x557e1966fdc0 is done 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967e4b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 10 trimming bilog shard 1 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000002.12.6 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 10 trimming bilog shard 3 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.5.6 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 10 trimming bilog shard 5 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.4.6 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 10 trimming bilog shard 6 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000002.12.6 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 10 trimming bilog shard 7 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.8.6 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 10 trimming bilog shard 8 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.14.6 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 10 trimming bilog shard 9 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.9.6 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 run: stack=0x557e1957ead0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 collect(): s=0x557e1957ee80 stack=0x557e1966fdc0 is complete 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) markers=[,,,,,00000000003.17.10,,,00000000031.93.9,,], shards=11 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e19684590:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e196821f0:op=0x557e196b0b20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196b8fe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196b4510:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196bb470:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e196c65a0:op=0x557e196c9780:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e192c4fb0:op=0x557e196c18f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196bea00:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 10 trimming bilog shard 5 of :dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) at marker 00000000003.17.10 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 10 trimming bilog shard 8 of :dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) at marker 00000000031.93.9 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e19684590:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.1 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is io blocked 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e196821f0:op=0x557e196b0b20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.3 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 run: stack=0x557e196821f0 is io blocked 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196b8fe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.5 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 run: stack=0x557e196ba420 is io blocked 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196b4510:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.6 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 run: stack=0x557e196d7810 is io blocked 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196bb470:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.7 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 run: stack=0x557e196baa80 is io blocked 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e196c65a0:op=0x557e196c9780:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.8 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 run: stack=0x557e196c65a0 is io blocked 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e192c4fb0:op=0x557e196c18f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.9 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 run: stack=0x557e192c4fb0 is io blocked 2026-03-20T11:45:37.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e1966fdc0:op=0x557e196bc230:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e192c5980:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e1966fdc0:op=0x557e196bc230:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21.5 2026-03-20T11:45:37.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 run: stack=0x557e1966fdc0 is io blocked 2026-03-20T11:45:37.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e192c5980:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21.8 2026-03-20T11:45:37.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.391+0000 7f8f59593900 20 run: stack=0x557e196be570 is io blocked 2026-03-20T11:45:37.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f59593900 20 cr:s=0x557e196b8e90:op=0x557e19670c10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f59593900 20 cr:s=0x557e196b8e90:op=0x557e19670c10:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:37.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f59593900 20 cr:s=0x557e196b8e90:op=0x557e1967f7b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f59593900 15 stack 0x557e196b8e90 end 2026-03-20T11:45:37.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f59593900 20 run: stack=0x557e196b8e90 is done 2026-03-20T11:45:37.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f59593900 20 collect(): s=0x557e196ad300 stack=0x557e196b8e90 is complete 2026-03-20T11:45:37.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f59593900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:37.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f59593900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:37.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:37.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f59593900 15 stack 0x557e196ad300 end 2026-03-20T11:45:37.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f59593900 20 stack->operate() returned ret=-22 2026-03-20T11:45:37.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f59593900 20 run: stack=0x557e196ad300 is done 2026-03-20T11:45:37.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f59593900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:37.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f59593900 4 starting trim on bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:45:37.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196cb0a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196cb0a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f59593900 20 enqueued request req=0x557e1967f430 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f59593900 20 req: 0x557e1967f430 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f59593900 20 run: stack=0x557e196ad300 is io blocked 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f0ffff640 20 dequeued request req=0x557e1967f430 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f0ffff640 20 RGWWQ: empty 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f0ffff640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 : miss 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.393+0000 7f8f0ffff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e192c5980:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e192c5980:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e192c5980:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e192c5980:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 15 stack 0x557e196be570 end 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 run: stack=0x557e196be570 is done 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196bea00:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e19684590:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e19684590:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196bb470:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e19684590:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196bb470:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e19684590:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 15 stack 0x557e1967dc20 end 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is done 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196bb470:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967e4b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 run: stack=0x557e1957ead0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196bb470:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 15 stack 0x557e196baa80 end 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 run: stack=0x557e196baa80 is done 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967e4b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 run: stack=0x557e1957ead0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f0ffff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f0ffff640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 info.flags=0x17 2026-03-20T11:45:37.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f0ffff640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 to cache LRU end 2026-03-20T11:45:37.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f0ffff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f0ffff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 cr:s=0x557e196821f0:op=0x557e196b0b20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 cr:s=0x557e196821f0:op=0x557e196b0b20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 cr:s=0x557e196821f0:op=0x557e196b0b20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 cr:s=0x557e196821f0:op=0x557e196b0b20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 15 stack 0x557e196821f0 end 2026-03-20T11:45:37.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 run: stack=0x557e196821f0 is done 2026-03-20T11:45:37.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967e4b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 run: stack=0x557e1957ead0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f0ffff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:45:37.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f0ffff640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-3 : miss 2026-03-20T11:45:37.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196b4510:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196b4510:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196b4510:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f0ffff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196b4510:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 15 stack 0x557e196d7810 end 2026-03-20T11:45:37.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 run: stack=0x557e196d7810 is done 2026-03-20T11:45:37.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967e4b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 run: stack=0x557e1957ead0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.394+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196b4f10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f0ffff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f0ffff640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-3 info.flags=0x0 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f0ffff640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-3 to cache LRU end 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f0ffff640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-3 : miss 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f0ffff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196b4f10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196b8fe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e192c4fb0:op=0x557e196c18f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e1966fdc0:op=0x557e196bc230:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e196c65a0:op=0x557e196c9780:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196b4f10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196b8fe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e192c4fb0:op=0x557e196c18f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e1966fdc0:op=0x557e196bc230:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e196c65a0:op=0x557e196c9780:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196b4f10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196b8fe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e192c4fb0:op=0x557e196c18f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e1966fdc0:op=0x557e196bc230:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e196c65a0:op=0x557e196c9780:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196bfcc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 15 stack 0x557e19688770 end 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 run: stack=0x557e19688770 is done 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196b8fe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 15 stack 0x557e196ba420 end 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f0ffff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 run: stack=0x557e196ba420 is done 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e192c4fb0:op=0x557e196c18f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 15 stack 0x557e192c4fb0 end 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f0ffff640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-3 info.flags=0x0 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 run: stack=0x557e192c4fb0 is done 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e1966fdc0:op=0x557e196bc230:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 15 stack 0x557e1966fdc0 end 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 run: stack=0x557e1966fdc0 is done 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f0ffff640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-3 to cache LRU end 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e196c65a0:op=0x557e196c9780:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 15 stack 0x557e196c65a0 end 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 run: stack=0x557e196c65a0 is done 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 collect(): s=0x557e196700a0 stack=0x557e19688770 is complete 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-10[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15]) markers=[,00000000002.9.6,,,,,,,,,], shards=11 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f0ffff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding source pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967e4b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196bea00:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e192c8420:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 10 trimming bilog shard 1 of :dipepu-10[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15]) at marker 00000000002.9.6 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 run: stack=0x557e196700a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967e4b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 15 stack 0x557e1957ee80 end 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is done 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196b4510:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967e4b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196b4510:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15.1 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f0ffff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding dest pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 run: stack=0x557e196ba420 is io blocked 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 15 stack 0x557e1957ead0 end 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 run: stack=0x557e1957ead0 is done 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 4 starting trim on bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196be890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 4 starting trim on bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196be890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 enqueued request req=0x557e1966faf0 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 req: 0x557e1966faf0 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is io blocked 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e196c6a50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e196c6a50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 enqueued request req=0x557e19681af0 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 req: 0x557e1966faf0 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 req: 0x557e19681af0 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 run: stack=0x557e1957ead0 is io blocked 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f0e7fc640 20 dequeued request req=0x557e1966faf0 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f0e7fc640 20 RGWWQ: 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f0e7fc640 20 req: 0x557e19681af0 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f0e7fc640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 : miss 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f0e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f0effd640 20 dequeued request req=0x557e19681af0 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f0effd640 20 RGWWQ: empty 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f0ffff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f0ffff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f0effd640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 : miss 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f0effd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f0ffff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196cb0a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196cb0a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196cb0a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196cb0a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196ccf80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e19670c10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e19670c10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.395 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.395+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 6864fd7a23edfe16d1214cd1466aee76cf6d0cc7f9f8b81c6fe1ad22cfb011b7 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:6864fd7a23edfe16d1214cd1466aee76cf6d0cc7f9f8b81c6fe1ad22cfb011b7 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 391e7fc37d48865292aa7cde4700afe802f3950a52a2b8842a3fd3e7caf360d2 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=391e7fc37d48865292aa7cde4700afe802f3950a52a2b8842a3fd3e7caf360d2 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=8, curl_handle=0x557e196861c0 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f59593900 20 run: stack=0x557e19688770 is io blocked 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0effd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0effd640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 info.flags=0x17 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0effd640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 to cache LRU end 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0effd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0effd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0effd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0effd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-38 : miss 2026-03-20T11:45:37.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0effd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0e7fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:45:37.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0e7fc640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 info.flags=0x17 2026-03-20T11:45:37.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0e7fc640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 to cache LRU end 2026-03-20T11:45:37.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0e7fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0e7fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0e7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:45:37.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0e7fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-9 : miss 2026-03-20T11:45:37.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8eb2ffd640 20 link_request req_data=0x557e196868d0 req_data->id=8, curl_handle=0x557e196861c0 2026-03-20T11:45:37.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0effd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0effd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-38 info.flags=0x0 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0effd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-38 to cache LRU end 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0effd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-38 : miss 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0effd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0e7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0e7fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-9 info.flags=0x0 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0e7fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-9 to cache LRU end 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0e7fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-9 : miss 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0effd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0effd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-38 info.flags=0x0 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0effd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-38 to cache LRU end 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0effd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding source pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0effd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding dest pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0effd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0effd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.396+0000 7f8f0effd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e196c6a50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e196c6a50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e196c6a50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e196c6a50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e1967e480:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 run: stack=0x557e1957ead0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196cb0a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196cb0a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f0e7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f0e7fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-9 info.flags=0x0 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f0e7fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-9 to cache LRU end 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f0e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding source pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f0e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding dest pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = eda1fe9671891d10bc924c1b822da762b6ca563e258c255753d535337f804f68 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:eda1fe9671891d10bc924c1b822da762b6ca563e258c255753d535337f804f68 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f0e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f0e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 8aa64ebf205a5ea7e235303427708b178b8512aa9e858df212064c1d8b07c140 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=8aa64ebf205a5ea7e235303427708b178b8512aa9e858df212064c1d8b07c140 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=9, curl_handle=0x557e196b2160 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 run: stack=0x557e196d7810 is io blocked 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196b4510:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196b4510:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196b4510:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196b4510:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 15 stack 0x557e196ba420 end 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 run: stack=0x557e196ba420 is done 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e192c8420:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f0e7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 15 stack 0x557e196700a0 end 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 run: stack=0x557e196700a0 is done 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 4 starting trim on bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c6a50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c6a50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 enqueued request req=0x557e19670280 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 req: 0x557e19670280 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 run: stack=0x557e196700a0 is io blocked 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196be890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196be890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196be890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196be890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e19683e10:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196be890:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196be890:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8eb2ffd640 20 link_request req_data=0x557e196b2870 req_data->id=9, curl_handle=0x557e196b2160 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.399 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = e81c59ade064e8cde6410ee8559f8d0dfe99d310e12e11594b83170710dc2a3f 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:e81c59ade064e8cde6410ee8559f8d0dfe99d310e12e11594b83170710dc2a3f 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 9c04ef45fface65de1a0babebda1bd366ebd3f174c5de2c5fe63a0078fa4c30c 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=9c04ef45fface65de1a0babebda1bd366ebd3f174c5de2c5fe63a0078fa4c30c 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=10, curl_handle=0x557e196b6260 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f59593900 20 run: stack=0x557e196ba420 is io blocked 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8eb2ffd640 20 link_request req_data=0x557e196b6970 req_data->id=10, curl_handle=0x557e196b6260 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f0dffb640 20 dequeued request req=0x557e19670280 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f0dffb640 20 RGWWQ: empty 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f0dffb640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 : miss 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f0dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f0dffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f0dffb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 info.flags=0x17 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f0dffb640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 to cache LRU end 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f0dffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f0dffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f0dffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f0dffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-15 : miss 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.397+0000 7f8f0dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.398+0000 7f8f0dffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.398+0000 7f8f0dffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-15 info.flags=0x0 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.398+0000 7f8f0dffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-15 to cache LRU end 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.398+0000 7f8f0dffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-15 : miss 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.398+0000 7f8f0dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.398+0000 7f8f0dffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.398+0000 7f8f0dffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-15 info.flags=0x0 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.398+0000 7f8f0dffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-15 to cache LRU end 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.398+0000 7f8f0dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding source pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.398+0000 7f8f0dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding dest pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f0dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f0dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f0dffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c6a50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c6a50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c6a50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c6a50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 20 cr:s=0x557e196821f0:op=0x557e196c6a50:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 20 run: stack=0x557e196700a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 20 cr:s=0x557e196821f0:op=0x557e196b4510:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 20 cr:s=0x557e196821f0:op=0x557e196b4510:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 2924ade1368b1f64294a9059a4c3716a310e6c50bf38ca646bf84137db86a286 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:2924ade1368b1f64294a9059a4c3716a310e6c50bf38ca646bf84137db86a286 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = f6d885dad02a282cc970732344ae1717c516575ac009f7ce6810b9ef089452e5 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=f6d885dad02a282cc970732344ae1717c516575ac009f7ce6810b9ef089452e5 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=11, curl_handle=0x557e19681800 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8f59593900 20 run: stack=0x557e196821f0 is io blocked 2026-03-20T11:45:37.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.399+0000 7f8eb2ffd640 20 link_request req_data=0x557e196823d0 req_data->id=11, curl_handle=0x557e19681800 2026-03-20T11:45:37.401 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:37.398+0000 7fd019a81640 -1 req 4758409563036738158 0.003000005s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e19670c10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e19670c10:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196ccf80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f59593900 15 stack 0x557e19688770 end 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f59593900 20 run: stack=0x557e19688770 is done 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f59593900 20 collect(): s=0x557e196ad300 stack=0x557e19688770 is complete 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f59593900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f59593900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f59593900 15 stack 0x557e196ad300 end 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f59593900 20 stack->operate() returned ret=-22 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f59593900 20 run: stack=0x557e196ad300 is done 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f59593900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f59593900 4 starting trim on bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1967fe70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1967fe70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f59593900 20 enqueued request req=0x557e1967f430 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f59593900 20 req: 0x557e1967f430 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f59593900 20 run: stack=0x557e196ad300 is io blocked 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f0d7fa640 20 dequeued request req=0x557e1967f430 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f0d7fa640 20 RGWWQ: empty 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f0d7fa640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 : miss 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f0d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f0d7fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f0d7fa640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 info.flags=0x17 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f0d7fa640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 to cache LRU end 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f0d7fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f0d7fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f0d7fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f0d7fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-13 : miss 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f0d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f0d7fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f0d7fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-13 info.flags=0x0 2026-03-20T11:45:37.437 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f0d7fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-13 to cache LRU end 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f0d7fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-13 : miss 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.439+0000 7f8f0d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f0d7fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f0d7fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-13 info.flags=0x0 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f0d7fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-13 to cache LRU end 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f0d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18): adding source pipe: {s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f0d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18): adding dest pipe: {s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f0d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f0d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f0d7fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1967fe70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1967fe70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1967fe70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1967fe70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e19670c10:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e192c83d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e192c83d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.438 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.439 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.439 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.439 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.439 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.439 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 33a16eff3fab5cd1bad5c507a12fe87a3fcbe98c3d405ca0978e746417b5ec80 2026-03-20T11:45:37.439 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.439 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.439 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.439 INFO:teuthology.orchestra.run.vm02.stderr:33a16eff3fab5cd1bad5c507a12fe87a3fcbe98c3d405ca0978e746417b5ec80 2026-03-20T11:45:37.439 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.439 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.439 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.439 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.439 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = da8a89b2eac415291bebb6e1fc42f29e8e3d3a5048efae0bc09c436b21503f41 2026-03-20T11:45:37.439 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=da8a89b2eac415291bebb6e1fc42f29e8e3d3a5048efae0bc09c436b21503f41 2026-03-20T11:45:37.439 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.439 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.439 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.439 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=12, curl_handle=0x557e196861c0 2026-03-20T11:45:37.439 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8f59593900 20 run: stack=0x557e19688770 is io blocked 2026-03-20T11:45:37.439 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.440+0000 7f8eb2ffd640 20 link_request req_data=0x557e196868d0 req_data->id=12, curl_handle=0x557e196861c0 2026-03-20T11:45:37.439 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.442+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196be890:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.440 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.442+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196be890:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.440 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.442+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196be890:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.440 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.442+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196be890:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.440 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.442+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e19683e10:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.440 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.442+0000 7f8f59593900 15 stack 0x557e196ba420 end 2026-03-20T11:45:37.440 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.442+0000 7f8f59593900 20 run: stack=0x557e196ba420 is done 2026-03-20T11:45:37.440 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.442+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.440 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.442+0000 7f8f59593900 20 collect(): s=0x557e1957ee80 stack=0x557e196ba420 is complete 2026-03-20T11:45:37.440 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.442+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.440 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.442+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-9[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14]) markers=[,00000000002.13.6,,,,,,,,,], shards=11 2026-03-20T11:45:37.440 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.442+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196c4440:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.440 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.442+0000 7f8f59593900 10 trimming bilog shard 1 of :dipepu-9[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14]) at marker 00000000002.13.6 2026-03-20T11:45:37.440 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.442+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.440 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.442+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196b4f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.440 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.442+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196b4f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.440 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.442+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14.1 2026-03-20T11:45:37.440 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.442+0000 7f8f59593900 20 run: stack=0x557e196ba420 is io blocked 2026-03-20T11:45:37.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.443+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196b4f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.443+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196b4f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.443+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196b4f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.443+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196b4f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.443+0000 7f8f59593900 15 stack 0x557e196ba420 end 2026-03-20T11:45:37.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.443+0000 7f8f59593900 20 run: stack=0x557e196ba420 is done 2026-03-20T11:45:37.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.443+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196c4440:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.443+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.443+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:45:37.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.443+0000 7f8f59593900 15 stack 0x557e1957ee80 end 2026-03-20T11:45:37.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.443+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is done 2026-03-20T11:45:37.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.443+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.443+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.443+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.443+0000 7f8f59593900 4 starting trim on bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:45:37.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.443+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e192c76a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.443+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e192c76a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.443+0000 7f8f59593900 20 enqueued request req=0x557e1966faf0 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.443+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.443+0000 7f8f59593900 20 req: 0x557e1966faf0 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.443+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is io blocked 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.443+0000 7f8f0cff9640 20 dequeued request req=0x557e1966faf0 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.443+0000 7f8f0cff9640 20 RGWWQ: empty 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.443+0000 7f8f0cff9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 : miss 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.443+0000 7f8f0cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196cb0a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f0cff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f0cff9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 info.flags=0x17 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f0cff9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 to cache LRU end 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f0cff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f0cff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f0cff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f0cff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-11 : miss 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f0cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196cb0a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 20 cr:s=0x557e196821f0:op=0x557e196b4510:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196cb0a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 20 cr:s=0x557e196821f0:op=0x557e196b4510:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196cb0a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 20 cr:s=0x557e196821f0:op=0x557e196b4510:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e1967e480:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 15 stack 0x557e196d7810 end 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 20 run: stack=0x557e196d7810 is done 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 20 cr:s=0x557e196821f0:op=0x557e196b4510:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 20 collect(): s=0x557e1957ead0 stack=0x557e196d7810 is complete 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-38[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43]) markers=[,,,,,,,,00000000001.38.6,,], shards=11 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 20 cr:s=0x557e196821f0:op=0x557e196c6a50:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 15 stack 0x557e196821f0 end 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 20 run: stack=0x557e196821f0 is done 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e196cb0a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 10 trimming bilog shard 8 of :dipepu-38[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43]) at marker 00000000001.38.6 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 20 run: stack=0x557e1957ead0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 20 collect(): s=0x557e196700a0 stack=0x557e196821f0 is complete 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-15[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20]) markers=[,,,,,,,,00000000007.20.6,,], shards=11 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196b4510:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967e480:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 10 trimming bilog shard 8 of :dipepu-15[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20]) at marker 00000000007.20.6 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 20 run: stack=0x557e196700a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196b4510:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43.8 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 20 run: stack=0x557e196d7810 is io blocked 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 20 cr:s=0x557e196821f0:op=0x557e196b4f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 20 cr:s=0x557e196821f0:op=0x557e196b4f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20.8 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.444+0000 7f8f59593900 20 run: stack=0x557e196821f0 is io blocked 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f0cff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f0cff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-11 info.flags=0x0 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f0cff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-11 to cache LRU end 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f0cff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-11 : miss 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f0cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f0cff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f0cff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-11 info.flags=0x0 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f0cff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-11 to cache LRU end 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f0cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16): adding source pipe: {s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f0cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16): adding dest pipe: {s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f0cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f0cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f0cff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e192c76a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e192c76a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e192c76a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e192c76a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196c1e10:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196cbfe0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196cbfe0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 05152dbb2bbdba435aceefd5d6051e840b6d194af2d2163483eef920cee45646 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:05152dbb2bbdba435aceefd5d6051e840b6d194af2d2163483eef920cee45646 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = c6a8e85d7a6232701483b8d2cfe42e4c1360c26680b0388390cdb61afb06fc69 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c6a8e85d7a6232701483b8d2cfe42e4c1360c26680b0388390cdb61afb06fc69 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=13, curl_handle=0x557e19681800 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8f59593900 20 run: stack=0x557e196ba420 is io blocked 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.445+0000 7f8eb2ffd640 20 link_request req_data=0x557e196823d0 req_data->id=13, curl_handle=0x557e19681800 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196b4510:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196b4510:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196b4510:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196b4510:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 15 stack 0x557e196d7810 end 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 run: stack=0x557e196d7810 is done 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e196cb0a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 15 stack 0x557e1957ead0 end 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 run: stack=0x557e1957ead0 is done 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 4 starting trim on bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e19684590:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e19684590:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 enqueued request req=0x557e19671a40 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 req: 0x557e19671a40 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 run: stack=0x557e1957ead0 is io blocked 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f0c7f8640 20 dequeued request req=0x557e19671a40 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f0c7f8640 20 RGWWQ: empty 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f0c7f8640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 : miss 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f0c7f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 cr:s=0x557e196821f0:op=0x557e196b4f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 cr:s=0x557e196821f0:op=0x557e196b4f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 cr:s=0x557e196821f0:op=0x557e196b4f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 cr:s=0x557e196821f0:op=0x557e196b4f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 15 stack 0x557e196821f0 end 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 run: stack=0x557e196821f0 is done 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967e480:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 15 stack 0x557e196700a0 end 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 run: stack=0x557e196700a0 is done 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 4 starting trim on bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c6a50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c6a50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 enqueued request req=0x557e19681af0 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 req: 0x557e19681af0 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f59593900 20 run: stack=0x557e196700a0 is io blocked 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f0bff7640 20 dequeued request req=0x557e19681af0 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f0bff7640 20 RGWWQ: empty 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f0bff7640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 : miss 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.446+0000 7f8f0bff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0c7f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0c7f8640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 info.flags=0x17 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0c7f8640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 to cache LRU end 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0c7f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0c7f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0c7f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0c7f8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-12 : miss 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0c7f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0bff7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0bff7640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 info.flags=0x17 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0bff7640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 to cache LRU end 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0bff7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0bff7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0bff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0bff7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-40 : miss 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0bff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0bff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0bff7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-40 info.flags=0x0 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0bff7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-40 to cache LRU end 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0bff7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-40 : miss 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0bff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0c7f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0c7f8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-12 info.flags=0x0 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0c7f8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-12 to cache LRU end 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0c7f8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-12 : miss 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0c7f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0c7f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0c7f8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-12 info.flags=0x0 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0c7f8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-12 to cache LRU end 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0c7f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17): adding source pipe: {s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0c7f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17): adding dest pipe: {s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0c7f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0c7f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f0c7f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f0bff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f0bff7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-40 info.flags=0x0 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f0bff7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-40 to cache LRU end 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f0bff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1): adding source pipe: {s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f0bff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1): adding dest pipe: {s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f0bff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f0bff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f0bff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.447+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e19684590:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e19684590:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c6a50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e19684590:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c6a50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e19684590:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c6a50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c6a50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 cr:s=0x557e196821f0:op=0x557e196c9780:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 run: stack=0x557e1957ead0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 cr:s=0x557e196821f0:op=0x557e196c4310:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e19684590:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 run: stack=0x557e196700a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 cr:s=0x557e196821f0:op=0x557e196c4310:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = e8db7e79d6e7945f6b8c29a15f42012a7abd2ec00e0a8b43f77bf20e5b330f3b 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:e8db7e79d6e7945f6b8c29a15f42012a7abd2ec00e0a8b43f77bf20e5b330f3b 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 512144f46bb3ca71255824aa441547432c7eb8f737efc96749d5bffd4a11c259 2026-03-20T11:45:37.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=512144f46bb3ca71255824aa441547432c7eb8f737efc96749d5bffd4a11c259 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=14, curl_handle=0x557e196b2160 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 run: stack=0x557e196821f0 is io blocked 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196c6a50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196c6a50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 348cbb3a2a6252ee6d55072da7ff9a446c3413349b40c500241a161783c1cacb 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:348cbb3a2a6252ee6d55072da7ff9a446c3413349b40c500241a161783c1cacb 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = c4059c5ecdd2d494d0eaa1f3e6362dfed13fa28d45d8c564b68b2e12d852da82 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c4059c5ecdd2d494d0eaa1f3e6362dfed13fa28d45d8c564b68b2e12d852da82 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=15, curl_handle=0x557e196b6260 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8f59593900 20 run: stack=0x557e196d7810 is io blocked 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8eb2ffd640 20 link_request req_data=0x557e196b2870 req_data->id=14, curl_handle=0x557e196b2160 2026-03-20T11:45:37.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.448+0000 7f8eb2ffd640 20 link_request req_data=0x557e196b6970 req_data->id=15, curl_handle=0x557e196b6260 2026-03-20T11:45:37.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.484+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e192c83d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.484+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e192c83d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.484+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e192c83d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.484+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e192c83d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.484+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e19670c10:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.484+0000 7f8f59593900 15 stack 0x557e19688770 end 2026-03-20T11:45:37.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.484+0000 7f8f59593900 20 run: stack=0x557e19688770 is done 2026-03-20T11:45:37.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.484+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.484+0000 7f8f59593900 20 collect(): s=0x557e196ad300 stack=0x557e19688770 is complete 2026-03-20T11:45:37.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.484+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.484+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) markers=[,,,,,00000000003.29.10,,,00000000003.11.6,00000000011.37.6,], shards=11 2026-03-20T11:45:37.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.484+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196cb0a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.484+0000 7f8f59593900 10 trimming bilog shard 5 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000003.29.10 2026-03-20T11:45:37.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.484+0000 7f8f59593900 10 trimming bilog shard 8 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000003.11.6 2026-03-20T11:45:37.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.484+0000 7f8f59593900 10 trimming bilog shard 9 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000011.37.6 2026-03-20T11:45:37.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.484+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.484+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196b4f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.484+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196d8570:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.484+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1967fe70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.484+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196b4f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.484+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.5 2026-03-20T11:45:37.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.484+0000 7f8f59593900 20 run: stack=0x557e19688770 is io blocked 2026-03-20T11:45:37.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.484+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196d8570:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.484+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.8 2026-03-20T11:45:37.483 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.484+0000 7f8f59593900 20 run: stack=0x557e196baa80 is io blocked 2026-03-20T11:45:37.483 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.484+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1967fe70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.483 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.484+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.9 2026-03-20T11:45:37.483 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.484+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is io blocked 2026-03-20T11:45:37.483 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196b4f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196b4f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196b4f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196b4f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 15 stack 0x557e19688770 end 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 run: stack=0x557e19688770 is done 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196cb0a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1967fe70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1967fe70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196d8570:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1967fe70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196d8570:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1967fe70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 15 stack 0x557e1967dc20 end 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is done 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196d8570:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196cb0a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196d8570:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 15 stack 0x557e196baa80 end 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 run: stack=0x557e196baa80 is done 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196cb0a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 15 stack 0x557e196ad300 end 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 run: stack=0x557e196ad300 is done 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 4 starting trim on bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196b44d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196b44d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 enqueued request req=0x557e19671360 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 req: 0x557e19671360 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f59593900 20 run: stack=0x557e196ad300 is io blocked 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f0b7f6640 20 dequeued request req=0x557e19671360 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f0b7f6640 20 RGWWQ: empty 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f0b7f6640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 : miss 2026-03-20T11:45:37.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f0b7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f0b7f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f0b7f6640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 info.flags=0x17 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f0b7f6640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 to cache LRU end 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f0b7f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f0b7f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f0b7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f0b7f6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-18 : miss 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.486+0000 7f8f0b7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f0b7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f0b7f6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-18 info.flags=0x0 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f0b7f6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-18 to cache LRU end 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f0b7f6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-18 : miss 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f0b7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f0b7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f0b7f6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-18 info.flags=0x0 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f0b7f6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-18 to cache LRU end 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f0b7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23): adding source pipe: {s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f0b7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23): adding dest pipe: {s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f0b7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f0b7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f0b7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196b44d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196b44d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196b44d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196b44d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e19670c10:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e19683e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e19683e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.485 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.486 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.486 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.486 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.486 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.486 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.486 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.486 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 56fbec16f3a6e55cf093f214dbe427268c0a4fd5dae83665df0a2ae06c5f0b58 2026-03-20T11:45:37.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.486 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.486 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.486 INFO:teuthology.orchestra.run.vm02.stderr:56fbec16f3a6e55cf093f214dbe427268c0a4fd5dae83665df0a2ae06c5f0b58 2026-03-20T11:45:37.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 6093405b9fbbe7496f2f915e9edd906aa2db37dd57b5b3ae0aea465e34b10a33 2026-03-20T11:45:37.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=6093405b9fbbe7496f2f915e9edd906aa2db37dd57b5b3ae0aea465e34b10a33 2026-03-20T11:45:37.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=16, curl_handle=0x557e196861c0 2026-03-20T11:45:37.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8f59593900 20 run: stack=0x557e196baa80 is io blocked 2026-03-20T11:45:37.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.487+0000 7f8eb2ffd640 20 link_request req_data=0x557e196868d0 req_data->id=16, curl_handle=0x557e196861c0 2026-03-20T11:45:37.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196cbfe0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196cbfe0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196cbfe0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196cbfe0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196c1e10:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 15 stack 0x557e196ba420 end 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 run: stack=0x557e196ba420 is done 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 collect(): s=0x557e1957ee80 stack=0x557e196ba420 is complete 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) markers=[00000000014.52.6,00000000008.34.6,00000000008.50.6,00000000008.32.6,00000000012.28.6,00000000010.31.6,00000000008.21.6,00000000010.27.6,00000000004.27.6,00000000008.22.6,00000000008.19.6], shards=11 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196cb0a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 10 trimming bilog shard 0 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000014.52.6 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 10 trimming bilog shard 1 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.34.6 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 10 trimming bilog shard 2 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.50.6 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 10 trimming bilog shard 3 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.32.6 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 10 trimming bilog shard 4 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000012.28.6 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 10 trimming bilog shard 5 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000010.31.6 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 10 trimming bilog shard 6 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.21.6 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 10 trimming bilog shard 7 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000010.27.6 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 10 trimming bilog shard 8 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000004.27.6 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 10 trimming bilog shard 9 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.22.6 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 10 trimming bilog shard 10 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.19.6 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196b4f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e192c7650:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196cbfe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196c8b90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196d2490:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 cr:s=0x557e196b9fa0:op=0x557e196d1740:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 cr:s=0x557e196cda50:op=0x557e196c18f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 cr:s=0x557e19684ee0:op=0x557e196bffe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196d9220:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d3ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196b4f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.0 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 run: stack=0x557e196ba420 is io blocked 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e192c7650:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.1 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is io blocked 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.2 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 run: stack=0x557e19688770 is io blocked 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196cbfe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.3 2026-03-20T11:45:37.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 run: stack=0x557e196be570 is io blocked 2026-03-20T11:45:37.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196c8b90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.489+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.4 2026-03-20T11:45:37.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.490+0000 7f8f59593900 20 run: stack=0x557e196cf7c0 is io blocked 2026-03-20T11:45:37.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.490+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196d2490:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.490+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.5 2026-03-20T11:45:37.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.490+0000 7f8f59593900 20 run: stack=0x557e196bcde0 is io blocked 2026-03-20T11:45:37.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.490+0000 7f8f59593900 20 cr:s=0x557e196b9fa0:op=0x557e196d1740:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.490+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.6 2026-03-20T11:45:37.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.490+0000 7f8f59593900 20 run: stack=0x557e196b9fa0 is io blocked 2026-03-20T11:45:37.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.490+0000 7f8f59593900 20 cr:s=0x557e196cda50:op=0x557e196c18f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.490+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.7 2026-03-20T11:45:37.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.490+0000 7f8f59593900 20 run: stack=0x557e196cda50 is io blocked 2026-03-20T11:45:37.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.490+0000 7f8f59593900 20 cr:s=0x557e19684ee0:op=0x557e196bffe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.490+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.8 2026-03-20T11:45:37.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.490+0000 7f8f59593900 20 run: stack=0x557e19684ee0 is io blocked 2026-03-20T11:45:37.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.490+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196d9220:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.490+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.9 2026-03-20T11:45:37.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.490+0000 7f8f59593900 20 run: stack=0x557e196c3030 is io blocked 2026-03-20T11:45:37.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.490+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d3ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.490+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.10 2026-03-20T11:45:37.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.490+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is io blocked 2026-03-20T11:45:37.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.491+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e192c7650:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.491+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e192c7650:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.491+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e192c7650:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.491+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e192c7650:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.491+0000 7f8f59593900 15 stack 0x557e1967dc20 end 2026-03-20T11:45:37.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.491+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is done 2026-03-20T11:45:37.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.491+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196cb0a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.491+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.491+0000 7f8f59593900 20 cr:s=0x557e196b9fa0:op=0x557e196d1740:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.491+0000 7f8f59593900 20 cr:s=0x557e196b9fa0:op=0x557e196d1740:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.491+0000 7f8f59593900 20 cr:s=0x557e196b9fa0:op=0x557e196d1740:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.491+0000 7f8f59593900 20 cr:s=0x557e196b9fa0:op=0x557e196d1740:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.491+0000 7f8f59593900 15 stack 0x557e196b9fa0 end 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.491+0000 7f8f59593900 20 run: stack=0x557e196b9fa0 is done 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.491+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196cb0a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.491+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.491+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196c8b90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.491+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196c8b90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.491+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196c8b90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.491+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196c8b90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.491+0000 7f8f59593900 15 stack 0x557e196cf7c0 end 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.491+0000 7f8f59593900 20 run: stack=0x557e196cf7c0 is done 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.491+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196cb0a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.491+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196c6a50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196c6a50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d3ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196d9220:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196cbfe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196c6a50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d3ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196d9220:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196cbfe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196c6a50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d3ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196d9220:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196cbfe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e19684590:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 15 stack 0x557e196d7810 end 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 run: stack=0x557e196d7810 is done 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d3ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 15 stack 0x557e196c7af0 end 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is done 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196d9220:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 15 stack 0x557e196c3030 end 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 run: stack=0x557e196c3030 is done 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196cbfe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 15 stack 0x557e196be570 end 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 run: stack=0x557e196be570 is done 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 collect(): s=0x557e196700a0 stack=0x557e196d7810 is complete 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-40[4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1]) markers=[,,,,,00000000001.103.6,,,,,], shards=11 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196cb0a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c8f10:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 10 trimming bilog shard 5 of :dipepu-40[4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1]) at marker 00000000001.103.6 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 run: stack=0x557e196700a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196cb0a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d3ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196cb0a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d3ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1.5 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 run: stack=0x557e196be570 is io blocked 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196b4f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196b4f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196b4f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e196b4f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 15 stack 0x557e196ba420 end 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 run: stack=0x557e196ba420 is done 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196cb0a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 15 stack 0x557e19688770 end 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 run: stack=0x557e19688770 is done 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196cb0a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e19684ee0:op=0x557e196bffe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e19684ee0:op=0x557e196bffe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e19684ee0:op=0x557e196bffe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e19684ee0:op=0x557e196bffe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 15 stack 0x557e19684ee0 end 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 run: stack=0x557e19684ee0 is done 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196cb0a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196cda50:op=0x557e196c18f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196cda50:op=0x557e196c18f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196cda50:op=0x557e196c18f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e196cda50:op=0x557e196c18f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 15 stack 0x557e196cda50 end 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 run: stack=0x557e196cda50 is done 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196cb0a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.492+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e196821f0:op=0x557e196c4310:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e196821f0:op=0x557e196c4310:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e196821f0:op=0x557e196c4310:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e196821f0:op=0x557e196c4310:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e196821f0:op=0x557e196c9780:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 15 stack 0x557e196821f0 end 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 run: stack=0x557e196821f0 is done 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 collect(): s=0x557e1957ead0 stack=0x557e196821f0 is complete 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) markers=[00000000014.44.6,00000000008.52.6,00000000008.40.6,00000000008.21.6,00000000012.26.6,00000000010.26.6,00000000008.35.6,00000000010.32.6,00000000004.38.6,00000000008.22.6,00000000008.23.6], shards=11 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e196b4640:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 10 trimming bilog shard 0 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000014.44.6 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 10 trimming bilog shard 1 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.52.6 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 10 trimming bilog shard 2 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.40.6 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 10 trimming bilog shard 3 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.21.6 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 10 trimming bilog shard 4 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000012.26.6 2026-03-20T11:45:37.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 10 trimming bilog shard 5 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000010.26.6 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 10 trimming bilog shard 6 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.35.6 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 10 trimming bilog shard 7 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000010.32.6 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 10 trimming bilog shard 8 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000004.38.6 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 10 trimming bilog shard 9 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.22.6 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 10 trimming bilog shard 10 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.23.6 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 run: stack=0x557e1957ead0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e192c5950:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196ffe10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196c96d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196c18f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e1920a900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e192c6ab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e196d1860:op=0x557e196d8e00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e196c51b0:op=0x557e196cbef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e196bb100:op=0x557e196cc6f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e196c9500:op=0x557e196d02d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.0 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 run: stack=0x557e19688770 is io blocked 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e192c5950:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.1 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 run: stack=0x557e196ba420 is io blocked 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196ffe10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.2 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is io blocked 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196c96d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.3 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 run: stack=0x557e196c3030 is io blocked 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196c18f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.4 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 run: stack=0x557e196d7810 is io blocked 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e1920a900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.5 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 run: stack=0x557e196cf7c0 is io blocked 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e192c6ab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.6 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is io blocked 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e196d1860:op=0x557e196d8e00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.7 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 run: stack=0x557e196d1860 is io blocked 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e196c51b0:op=0x557e196cbef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.8 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 run: stack=0x557e196c51b0 is io blocked 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e196bb100:op=0x557e196cc6f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.9 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 run: stack=0x557e196bb100 is io blocked 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e196c9500:op=0x557e196d02d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.10 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 run: stack=0x557e196c9500 is io blocked 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196d2490:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196d2490:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196d2490:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196d2490:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 15 stack 0x557e196bcde0 end 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 run: stack=0x557e196bcde0 is done 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196cb0a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 15 stack 0x557e1957ee80 end 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is done 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 4 starting trim on bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196c3400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196c3400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 enqueued request req=0x557e196bf240 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 req: 0x557e196bf240 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.493+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is io blocked 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.494+0000 7f8f0aff5640 20 dequeued request req=0x557e196bf240 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.494+0000 7f8f0aff5640 20 RGWWQ: empty 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.494+0000 7f8f0aff5640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 : miss 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.494+0000 7f8f0aff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196ffe10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196ffe10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196ffe10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196ffe10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 15 stack 0x557e196c7af0 end 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is done 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e196b4640:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 run: stack=0x557e1957ead0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 15 stack 0x557e19688770 end 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 run: stack=0x557e19688770 is done 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e196b4640:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 run: stack=0x557e1957ead0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f0aff5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f0aff5640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 info.flags=0x17 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f0aff5640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 to cache LRU end 2026-03-20T11:45:37.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f0aff5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f0aff5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196d1860:op=0x557e196d8e00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f0aff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196d1860:op=0x557e196d8e00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d3ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196d1860:op=0x557e196d8e00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d3ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196d1860:op=0x557e196d8e00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 15 stack 0x557e196d1860 end 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 run: stack=0x557e196d1860 is done 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d3ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e196b4640:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 run: stack=0x557e1957ead0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d3ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 15 stack 0x557e196be570 end 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 run: stack=0x557e196be570 is done 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c8f10:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f0aff5640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-50 : miss 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 15 stack 0x557e196700a0 end 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 run: stack=0x557e196700a0 is done 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f0aff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 4 starting trim on bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 enqueued request req=0x557e196beae0 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 req: 0x557e196beae0 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 run: stack=0x557e196700a0 is io blocked 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e1920a900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e1920a900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e192c6ab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196bb100:op=0x557e196cc6f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196c96d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e1920a900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e192c6ab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196bb100:op=0x557e196cc6f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196c96d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e1920a900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 15 stack 0x557e196cf7c0 end 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 run: stack=0x557e196cf7c0 is done 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e192c6ab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196bb100:op=0x557e196cc6f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196c96d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e196b4640:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 run: stack=0x557e1957ead0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e192c6ab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 15 stack 0x557e1967dc20 end 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is done 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196bb100:op=0x557e196cc6f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 15 stack 0x557e196bb100 end 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 run: stack=0x557e196bb100 is done 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196c96d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 15 stack 0x557e196c3030 end 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 run: stack=0x557e196c3030 is done 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e196b4640:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e196b4640:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e196b4640:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 run: stack=0x557e1957ead0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f0a7f4640 20 dequeued request req=0x557e196beae0 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f0a7f4640 20 RGWWQ: empty 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f0a7f4640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 : miss 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f0a7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e192c5950:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e192c5950:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e192c5950:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e196ba420:op=0x557e192c5950:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 15 stack 0x557e196ba420 end 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 run: stack=0x557e196ba420 is done 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e196b4640:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.495+0000 7f8f59593900 20 run: stack=0x557e1957ead0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196c18f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196c18f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196c18f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e196d7810:op=0x557e196c18f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 15 stack 0x557e196d7810 end 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 run: stack=0x557e196d7810 is done 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e196b4640:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 run: stack=0x557e1957ead0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e196c51b0:op=0x557e196cbef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e196c51b0:op=0x557e196cbef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e196c51b0:op=0x557e196cbef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e196c51b0:op=0x557e196cbef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 15 stack 0x557e196c51b0 end 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 run: stack=0x557e196c51b0 is done 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e196b4640:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 run: stack=0x557e1957ead0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e196c9500:op=0x557e196d02d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e196c9500:op=0x557e196d02d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e196c9500:op=0x557e196d02d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e196c9500:op=0x557e196d02d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 15 stack 0x557e196c9500 end 2026-03-20T11:45:37.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 run: stack=0x557e196c9500 is done 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e196b4640:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e1957ead0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 15 stack 0x557e1957ead0 end 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 run: stack=0x557e1957ead0 is done 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 4 starting trim on bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e196c8f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e196c8f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 enqueued request req=0x557e19681e40 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 req: 0x557e19681e40 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is io blocked 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0aff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0aff5640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-50 info.flags=0x0 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0aff5640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-50 to cache LRU end 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0aff5640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-50 : miss 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0aff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f09ff3640 20 dequeued request req=0x557e19681e40 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f09ff3640 20 RGWWQ: empty 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f09ff3640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 : miss 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f09ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0a7f4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0a7f4640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 info.flags=0x17 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0a7f4640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 to cache LRU end 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0a7f4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0a7f4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0a7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0a7f4640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-24 : miss 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0a7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0aff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0aff5640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-50 info.flags=0x0 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0aff5640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-50 to cache LRU end 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0aff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10): adding source pipe: {s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0aff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10): adding dest pipe: {s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0aff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0aff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0aff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0a7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0a7f4640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-24 info.flags=0x0 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196c3400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196c3400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0a7f4640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-24 to cache LRU end 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196c3400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196c3400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0a7f4640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-24 : miss 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0a7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196d3310:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196b95b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196b95b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.496 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 8daf40504d25905681683e2296ccf5b7f05a4c7aa11df80b14d4052415a07167 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:8daf40504d25905681683e2296ccf5b7f05a4c7aa11df80b14d4052415a07167 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 2d1704cd1d46c27b77627aa1d175af9bfee0ea922eecf3a6e9565bc00a1fcb88 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2d1704cd1d46c27b77627aa1d175af9bfee0ea922eecf3a6e9565bc00a1fcb88 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=17, curl_handle=0x557e196b2160 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8eb2ffd640 20 link_request req_data=0x557e196b2870 req_data->id=17, curl_handle=0x557e196b2160 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f59593900 20 run: stack=0x557e196c3030 is io blocked 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0a7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0a7f4640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-24 info.flags=0x0 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0a7f4640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-24 to cache LRU end 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0a7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29): adding source pipe: {s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0a7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29): adding dest pipe: {s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0a7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0a7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f0a7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.496+0000 7f8f09ff3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f09ff3640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 info.flags=0x17 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f09ff3640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 to cache LRU end 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f09ff3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f09ff3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f09ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f09ff3640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-56 : miss 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196c87b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f09ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 run: stack=0x557e196700a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196da060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196da060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 8dc5740767af134ba370b6e888cc0d79c0767bb89311d1c5c2dc758fa86e2b31 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:8dc5740767af134ba370b6e888cc0d79c0767bb89311d1c5c2dc758fa86e2b31 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 07f10726042595f3dd7ea6f1ac1350165e19b4a5265bb1d8983f4ea56d44374b 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=07f10726042595f3dd7ea6f1ac1350165e19b4a5265bb1d8983f4ea56d44374b 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=18, curl_handle=0x557e196b6260 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8eb2ffd640 20 link_request req_data=0x557e196b6970 req_data->id=18, curl_handle=0x557e196b6260 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 run: stack=0x557e196cf7c0 is io blocked 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f09ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f09ff3640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-56 info.flags=0x0 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f09ff3640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-56 to cache LRU end 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f09ff3640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-56 : miss 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f09ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f09ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f09ff3640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-56 info.flags=0x0 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f09ff3640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-56 to cache LRU end 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f09ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16): adding source pipe: {s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f09ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16): adding dest pipe: {s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f09ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f09ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f09ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e196c8f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e196c8f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e196c8f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e196c8f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e19684590:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.499 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:37.498+0000 7fd05a302640 -1 req 5323222469647121203 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:37.499 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:37.498+0000 7fd02a2a2640 -1 req 17803663745692189937 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196c8f10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196c8f10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 9745a635afbc634ea6cb4c8cfe24d66070d1b01e25779fdf96e5ab578ed697bd 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:9745a635afbc634ea6cb4c8cfe24d66070d1b01e25779fdf96e5ab578ed697bd 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 521cfc9b2c65db2094cf22ea6246e7a72a1badf87bb5802a194f31e634d5b38c 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=521cfc9b2c65db2094cf22ea6246e7a72a1badf87bb5802a194f31e634d5b38c 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=19, curl_handle=0x557e19681800 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8eb2ffd640 20 link_request req_data=0x557e196823d0 req_data->id=19, curl_handle=0x557e19681800 2026-03-20T11:45:37.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.497+0000 7f8f59593900 20 run: stack=0x557e196be570 is io blocked 2026-03-20T11:45:37.528 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.531+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e19683e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.531+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e19683e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.531+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e19683e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.531+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e19683e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.531+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e19670c10:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.531+0000 7f8f59593900 15 stack 0x557e196baa80 end 2026-03-20T11:45:37.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.531+0000 7f8f59593900 20 run: stack=0x557e196baa80 is done 2026-03-20T11:45:37.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.531+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.531+0000 7f8f59593900 20 collect(): s=0x557e196ad300 stack=0x557e196baa80 is complete 2026-03-20T11:45:37.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.531+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.531+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-18[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23]) markers=[,,,,,,,,00000000005.51.12,,], shards=11 2026-03-20T11:45:37.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.531+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19670c10:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.531+0000 7f8f59593900 10 trimming bilog shard 8 of :dipepu-18[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23]) at marker 00000000005.51.12 2026-03-20T11:45:37.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.531+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.531+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196d39c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.531+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196d39c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.531+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23.8 2026-03-20T11:45:37.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.531+0000 7f8f59593900 20 run: stack=0x557e196baa80 is io blocked 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196d39c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196d39c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196d39c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196d39c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f59593900 15 stack 0x557e196baa80 end 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f59593900 20 run: stack=0x557e196baa80 is done 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19670c10:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f59593900 15 stack 0x557e196ad300 end 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f59593900 20 run: stack=0x557e196ad300 is done 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f59593900 4 starting trim on bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1967fe70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1967fe70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f59593900 20 enqueued request req=0x557e19671360 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f59593900 20 req: 0x557e19671360 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f59593900 20 run: stack=0x557e196ad300 is io blocked 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f097f2640 20 dequeued request req=0x557e19671360 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f097f2640 20 RGWWQ: empty 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f097f2640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 : miss 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f097f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f097f2640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f097f2640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 info.flags=0x17 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f097f2640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 to cache LRU end 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f097f2640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f097f2640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.533+0000 7f8f097f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f097f2640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-27 : miss 2026-03-20T11:45:37.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f097f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f097f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f097f2640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-27 info.flags=0x0 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f097f2640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-27 to cache LRU end 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f097f2640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-27 : miss 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f097f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f097f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f097f2640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-27 info.flags=0x0 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f097f2640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-27 to cache LRU end 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f097f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32): adding source pipe: {s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f097f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32): adding dest pipe: {s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f097f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f097f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f097f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1967fe70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1967fe70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1967fe70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1967fe70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196bb270:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e19683e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e19683e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.533 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.533 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.533 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.533 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.533 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.533 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.533 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.533 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.533 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = fe5e8d6685e0771505263a6102a8ef4ed0a26bb1fc5900c4be927bdb1286e4f1 2026-03-20T11:45:37.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.533 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.533 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.533 INFO:teuthology.orchestra.run.vm02.stderr:fe5e8d6685e0771505263a6102a8ef4ed0a26bb1fc5900c4be927bdb1286e4f1 2026-03-20T11:45:37.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = d1b9b4f6f73c83d2d37857d123c60dc3797bfc1a7ed64a5982ff695e55cd6a18 2026-03-20T11:45:37.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=d1b9b4f6f73c83d2d37857d123c60dc3797bfc1a7ed64a5982ff695e55cd6a18 2026-03-20T11:45:37.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=20, curl_handle=0x557e196861c0 2026-03-20T11:45:37.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8eb2ffd640 20 link_request req_data=0x557e196868d0 req_data->id=20, curl_handle=0x557e196861c0 2026-03-20T11:45:37.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.534+0000 7f8f59593900 20 run: stack=0x557e196baa80 is io blocked 2026-03-20T11:45:37.534 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:37.535+0000 7fd08635a640 -1 req 16114465176076766276 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:37.535 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196b95b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196b95b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196b95b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196b95b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196d3310:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f59593900 15 stack 0x557e196c3030 end 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f59593900 20 run: stack=0x557e196c3030 is done 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f59593900 20 collect(): s=0x557e1957ee80 stack=0x557e196c3030 is complete 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f59593900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f59593900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f59593900 15 stack 0x557e1957ee80 end 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f59593900 20 stack->operate() returned ret=-22 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is done 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f59593900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f59593900 4 starting trim on bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e192c61f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e192c61f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f59593900 20 enqueued request req=0x557e196bf240 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f59593900 20 req: 0x557e196bf240 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is io blocked 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f08ff1640 20 dequeued request req=0x557e196bf240 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f08ff1640 20 RGWWQ: empty 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f08ff1640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 : miss 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f08ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f08ff1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=314 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f08ff1640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 info.flags=0x17 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f08ff1640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 to cache LRU end 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f08ff1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f08ff1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f08ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f08ff1640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-44 : miss 2026-03-20T11:45:37.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.538+0000 7f8f08ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f08ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f08ff1640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-44 info.flags=0x0 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f08ff1640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-44 to cache LRU end 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f08ff1640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-44 : miss 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f08ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196da060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196da060:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196c87b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 15 stack 0x557e196cf7c0 end 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 run: stack=0x557e196cf7c0 is done 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196c8f10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196c8f10:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 collect(): s=0x557e196700a0 stack=0x557e196cf7c0 is complete 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 15 stack 0x557e196700a0 end 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 stack->operate() returned ret=-22 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 run: stack=0x557e196700a0 is done 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e19684590:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 15 stack 0x557e196be570 end 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 run: stack=0x557e196be570 is done 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 collect(): s=0x557e1967dc20 stack=0x557e196be570 is complete 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1967b090:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 15 stack 0x557e1967dc20 end 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 stack->operate() returned ret=-22 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is done 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 4 starting trim on bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196d29b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 4 starting trim on bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196d29b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 enqueued request req=0x557e196ca3d0 2026-03-20T11:45:37.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 req: 0x557e196ca3d0 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 run: stack=0x557e196700a0 is io blocked 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1920a900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1920a900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 enqueued request req=0x557e192c8310 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 req: 0x557e196ca3d0 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 req: 0x557e192c8310 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is io blocked 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f087f0640 20 dequeued request req=0x557e196ca3d0 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f087f0640 20 RGWWQ: 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f087f0640 20 req: 0x557e192c8310 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f087f0640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 : miss 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f087f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f07fef640 20 dequeued request req=0x557e192c8310 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f07fef640 20 RGWWQ: empty 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f07fef640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 : miss 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f07fef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f08ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f08ff1640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-44 info.flags=0x0 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f08ff1640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-44 to cache LRU end 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f08ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4): adding source pipe: {s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f08ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4): adding dest pipe: {s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f08ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f08ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f08ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f087f0640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f087f0640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 info.flags=0x17 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f087f0640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 to cache LRU end 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f087f0640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f087f0640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f087f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e192c61f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e192c61f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e192c61f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f087f0640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-17 : miss 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e192c61f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f087f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f07fef640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f07fef640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 info.flags=0x17 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f07fef640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 to cache LRU end 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f07fef640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f07fef640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f07fef640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d3310:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f07fef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f07fef640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-46 : miss 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f07fef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196b95b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196b95b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 8aa4f55b5a68ccac6cf2f7148f58a9e344d051901f1c0c90e29486fb35b334b0 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:8aa4f55b5a68ccac6cf2f7148f58a9e344d051901f1c0c90e29486fb35b334b0 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 5619dd76cd0aa71d4152ed078cc54b6b78ff2848d58e58892c3d69b9d6553b6c 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5619dd76cd0aa71d4152ed078cc54b6b78ff2848d58e58892c3d69b9d6553b6c 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=21, curl_handle=0x557e19681800 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.539+0000 7f8eb2ffd640 20 link_request req_data=0x557e196823d0 req_data->id=21, curl_handle=0x557e19681800 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f087f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f087f0640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-17 info.flags=0x0 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f087f0640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-17 to cache LRU end 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f087f0640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-17 : miss 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f087f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 20 run: stack=0x557e196be570 is io blocked 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f087f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f087f0640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-17 info.flags=0x0 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f087f0640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-17 to cache LRU end 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f087f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22): adding source pipe: {s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f087f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22): adding dest pipe: {s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f087f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f087f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f087f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196d29b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196d29b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196d29b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196d29b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196c2540:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 20 run: stack=0x557e196700a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196da060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196da060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = c0832e526208058529b0d1635cda08d22c60be24fb4de3e59e31f7599f207375 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:c0832e526208058529b0d1635cda08d22c60be24fb4de3e59e31f7599f207375 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 14c877a448d8cc23eb720f3d5100643a5b563c753f3ab36d0148546f09fe7423 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=14c877a448d8cc23eb720f3d5100643a5b563c753f3ab36d0148546f09fe7423 2026-03-20T11:45:37.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=22, curl_handle=0x557e196b6260 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8eb2ffd640 20 link_request req_data=0x557e196b6970 req_data->id=22, curl_handle=0x557e196b6260 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f07fef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f07fef640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-46 info.flags=0x0 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f07fef640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-46 to cache LRU end 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f07fef640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-46 : miss 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f07fef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f59593900 20 run: stack=0x557e196cf7c0 is io blocked 2026-03-20T11:45:37.541 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:37.540+0000 7fd08034e640 -1 req 1104991285762774582 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:37.541 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:37.542+0000 7fd097b7d640 -1 req 1265690993103570383 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f07fef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f07fef640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-46 info.flags=0x0 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f07fef640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-46 to cache LRU end 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.540+0000 7f8f07fef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6): adding source pipe: {s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f07fef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6): adding dest pipe: {s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f07fef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f07fef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f07fef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1920a900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1920a900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1920a900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1920a900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e19684590:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196b4490:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196b4490:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.542 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.542 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.542 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.542 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.542 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.542 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.542 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.542 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.542 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = af6d6a236b8b60b1c019c1b56a171d34a82074cfb2e4097c702c1c3327943cb6 2026-03-20T11:45:37.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.542 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.542 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.542 INFO:teuthology.orchestra.run.vm02.stderr:af6d6a236b8b60b1c019c1b56a171d34a82074cfb2e4097c702c1c3327943cb6 2026-03-20T11:45:37.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 8234f610e41a1b1a592f01da63ed32fdfb74a6c2a64b4cf499d2529f896db9c9 2026-03-20T11:45:37.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=8234f610e41a1b1a592f01da63ed32fdfb74a6c2a64b4cf499d2529f896db9c9 2026-03-20T11:45:37.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=23, curl_handle=0x557e196b2160 2026-03-20T11:45:37.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8eb2ffd640 20 link_request req_data=0x557e196b2870 req_data->id=23, curl_handle=0x557e196b2160 2026-03-20T11:45:37.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.541+0000 7f8f59593900 20 run: stack=0x557e196c3030 is io blocked 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e19683e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e19683e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196bb270:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f59593900 15 stack 0x557e196baa80 end 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f59593900 20 run: stack=0x557e196baa80 is done 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f59593900 20 collect(): s=0x557e196ad300 stack=0x557e196baa80 is complete 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f59593900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f59593900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f59593900 15 stack 0x557e196ad300 end 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f59593900 20 stack->operate() returned ret=-22 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f59593900 20 run: stack=0x557e196ad300 is done 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f59593900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f59593900 4 starting trim on bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1967fe70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1967fe70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f59593900 20 enqueued request req=0x557e19671360 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f59593900 20 req: 0x557e19671360 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f59593900 20 run: stack=0x557e196ad300 is io blocked 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f077ee640 20 dequeued request req=0x557e19671360 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f077ee640 20 RGWWQ: empty 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f077ee640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 : miss 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f077ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f077ee640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f077ee640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 info.flags=0x17 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f077ee640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 to cache LRU end 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f077ee640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f077ee640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f077ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f077ee640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-37 : miss 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.577+0000 7f8f077ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f077ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f077ee640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-37 info.flags=0x0 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f077ee640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-37 to cache LRU end 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f077ee640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-37 : miss 2026-03-20T11:45:37.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f077ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f077ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f077ee640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-37 info.flags=0x0 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f077ee640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-37 to cache LRU end 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f077ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42): adding source pipe: {s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f077ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42): adding dest pipe: {s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f077ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f077ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f077ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1967fe70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1967fe70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1967fe70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1967fe70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196bb270:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e19683e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e19683e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.576 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.577 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.577 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.577 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.577 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.577 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 8d271c97f1e2d69080de129291743eeb8b712844136748536ed269303d38292c 2026-03-20T11:45:37.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.577 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.577 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.577 INFO:teuthology.orchestra.run.vm02.stderr:8d271c97f1e2d69080de129291743eeb8b712844136748536ed269303d38292c 2026-03-20T11:45:37.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 9480695c9dc17e2a4899cc20f96924824bb613a1d2127ce7eecabfc0d3cf0405 2026-03-20T11:45:37.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=9480695c9dc17e2a4899cc20f96924824bb613a1d2127ce7eecabfc0d3cf0405 2026-03-20T11:45:37.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=24, curl_handle=0x557e196861c0 2026-03-20T11:45:37.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8eb2ffd640 20 link_request req_data=0x557e196868d0 req_data->id=24, curl_handle=0x557e196861c0 2026-03-20T11:45:37.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.578+0000 7f8f59593900 20 run: stack=0x557e196baa80 is io blocked 2026-03-20T11:45:37.579 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196b95b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196b95b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d3310:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f59593900 15 stack 0x557e196be570 end 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f59593900 20 run: stack=0x557e196be570 is done 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f59593900 20 collect(): s=0x557e1957ee80 stack=0x557e196be570 is complete 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f59593900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f59593900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f59593900 15 stack 0x557e1957ee80 end 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f59593900 20 stack->operate() returned ret=-22 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is done 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f59593900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f59593900 4 starting trim on bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e192c61f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e192c61f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f59593900 20 enqueued request req=0x557e196bf240 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f59593900 20 req: 0x557e196bf240 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is io blocked 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f06fed640 20 dequeued request req=0x557e196bf240 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f06fed640 20 RGWWQ: empty 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f06fed640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 : miss 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f06fed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f06fed640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f06fed640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 info.flags=0x17 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f06fed640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 to cache LRU end 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f06fed640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f06fed640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f06fed640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f06fed640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-1 : miss 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f06fed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f06fed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f06fed640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-1 info.flags=0x0 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f06fed640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-1 to cache LRU end 2026-03-20T11:45:37.580 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f06fed640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-1 : miss 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.582+0000 7f8f06fed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f06fed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f06fed640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-1 info.flags=0x0 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f06fed640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-1 to cache LRU end 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f06fed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1): adding source pipe: {s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f06fed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1): adding dest pipe: {s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f06fed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f06fed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f06fed640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e192c61f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e192c61f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e192c61f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e192c61f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d3310:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196b95b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196b95b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.581 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = d7e1d8cd1cd7767e4f4c15ce81e615b4cb0c662d822b754809e2fd255c4a822a 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:d7e1d8cd1cd7767e4f4c15ce81e615b4cb0c662d822b754809e2fd255c4a822a 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 8b3790c551461fbbc58bf71bdb89e7f839c8b25f74a9a95224932ed7036de728 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=8b3790c551461fbbc58bf71bdb89e7f839c8b25f74a9a95224932ed7036de728 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=25, curl_handle=0x557e19681800 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8eb2ffd640 20 link_request req_data=0x557e196823d0 req_data->id=25, curl_handle=0x557e19681800 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.583+0000 7f8f59593900 20 run: stack=0x557e196be570 is io blocked 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196da060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196da060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196b4490:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196b4490:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196da060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e19684590:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 15 stack 0x557e196c3030 end 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 run: stack=0x557e196c3030 is done 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196da060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 collect(): s=0x557e1967dc20 stack=0x557e196c3030 is complete 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1967b090:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 15 stack 0x557e1967dc20 end 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 stack->operate() returned ret=-22 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is done 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196c2540:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 15 stack 0x557e196cf7c0 end 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 run: stack=0x557e196cf7c0 is done 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 collect(): s=0x557e196700a0 stack=0x557e196cf7c0 is complete 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-17[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22]) markers=[,,,,,,,,00000000005.46.18,,], shards=11 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 4 starting trim on bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e19210c20:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 10 trimming bilog shard 8 of :dipepu-17[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22]) at marker 00000000005.46.18 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 run: stack=0x557e196700a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e19684590:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196d29b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e19684590:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 enqueued request req=0x557e192c8310 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 req: 0x557e192c8310 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is io blocked 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196d29b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22.8 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f067ec640 20 dequeued request req=0x557e192c8310 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f067ec640 20 RGWWQ: empty 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f067ec640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 : miss 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f067ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.584+0000 7f8f59593900 20 run: stack=0x557e196cf7c0 is io blocked 2026-03-20T11:45:37.583 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:37.584+0000 7fd0a2b93640 -1 req 3046217822034639448 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:37.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f067ec640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:45:37.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f067ec640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 info.flags=0x17 2026-03-20T11:45:37.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f067ec640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 to cache LRU end 2026-03-20T11:45:37.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f067ec640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f067ec640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f067ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:45:37.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f067ec640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-7 : miss 2026-03-20T11:45:37.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f067ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f067ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f067ec640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-7 info.flags=0x0 2026-03-20T11:45:37.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f067ec640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-7 to cache LRU end 2026-03-20T11:45:37.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f067ec640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-7 : miss 2026-03-20T11:45:37.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f067ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196d29b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196d29b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196d29b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196d29b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f59593900 15 stack 0x557e196cf7c0 end 2026-03-20T11:45:37.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f59593900 20 run: stack=0x557e196cf7c0 is done 2026-03-20T11:45:37.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e19210c20:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f59593900 15 stack 0x557e196700a0 end 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f59593900 20 run: stack=0x557e196700a0 is done 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f59593900 4 starting trim on bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196d29b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196d29b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f59593900 20 enqueued request req=0x557e196beae0 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f59593900 20 req: 0x557e196beae0 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f59593900 20 run: stack=0x557e196700a0 is io blocked 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f05feb640 20 dequeued request req=0x557e196beae0 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f05feb640 20 RGWWQ: empty 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f05feb640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 : miss 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.585+0000 7f8f05feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f067ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f067ec640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-7 info.flags=0x0 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f067ec640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-7 to cache LRU end 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f067ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12): adding source pipe: {s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f067ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12): adding dest pipe: {s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f067ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f067ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f067ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e19684590:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e19684590:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e19684590:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e19684590:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196c2540:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196da060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196da060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.585 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 8153fda302da1cf5ae21d8957066239f8a439832d3fcbaa9355a366088d4cb4c 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:8153fda302da1cf5ae21d8957066239f8a439832d3fcbaa9355a366088d4cb4c 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 23ea68a9f1204c6bd4fc07fcf3e2103293d2db48ca6ca8390f8f1ba320206e60 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=23ea68a9f1204c6bd4fc07fcf3e2103293d2db48ca6ca8390f8f1ba320206e60 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=26, curl_handle=0x557e196b2160 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8eb2ffd640 20 link_request req_data=0x557e196b2870 req_data->id=26, curl_handle=0x557e196b2160 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f05feb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f05feb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 info.flags=0x17 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f05feb640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 to cache LRU end 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f05feb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f05feb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f05feb640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 run: stack=0x557e196cf7c0 is io blocked 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f05feb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f05feb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-48 : miss 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f05feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f05feb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f05feb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-48 info.flags=0x0 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f05feb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-48 to cache LRU end 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f05feb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-48 : miss 2026-03-20T11:45:37.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f05feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.587 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:37.587+0000 7fd0a3394640 -1 req 8121655410108103455 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f05feb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f05feb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-48 info.flags=0x0 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f05feb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-48 to cache LRU end 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f05feb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8): adding source pipe: {s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f05feb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8): adding dest pipe: {s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f05feb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f05feb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f05feb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196d29b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196d29b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196d29b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196d29b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196b4490:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 run: stack=0x557e196700a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e19684590:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e19684590:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.586+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.587+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.587+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.587+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 6f53e084c9098947735ac88a35c1a79f1cb53b009ed7ec8bfdca3f10a3b20c32 2026-03-20T11:45:37.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.587+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.588 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.588 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.588 INFO:teuthology.orchestra.run.vm02.stderr:6f53e084c9098947735ac88a35c1a79f1cb53b009ed7ec8bfdca3f10a3b20c32 2026-03-20T11:45:37.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.587+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.587+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.587+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.587+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.587+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 4f163f85c71a46b3573d9e03e9f1ce98215afb9d95a6bbf901e1bee41d74e16d 2026-03-20T11:45:37.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.587+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=4f163f85c71a46b3573d9e03e9f1ce98215afb9d95a6bbf901e1bee41d74e16d 2026-03-20T11:45:37.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.587+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.587+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.587+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.587+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=27, curl_handle=0x557e196b6260 2026-03-20T11:45:37.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.587+0000 7f8eb2ffd640 20 link_request req_data=0x557e196b6970 req_data->id=27, curl_handle=0x557e196b6260 2026-03-20T11:45:37.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.587+0000 7f8f59593900 20 run: stack=0x557e196c3030 is io blocked 2026-03-20T11:45:37.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.622+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e19683e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.622+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e19683e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.622+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e19683e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.622+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e19683e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.622+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196bb270:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.622+0000 7f8f59593900 15 stack 0x557e196baa80 end 2026-03-20T11:45:37.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.622+0000 7f8f59593900 20 run: stack=0x557e196baa80 is done 2026-03-20T11:45:37.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.622+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.622+0000 7f8f59593900 20 collect(): s=0x557e196ad300 stack=0x557e196baa80 is complete 2026-03-20T11:45:37.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.622+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.622+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-37[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42]) markers=[,,,,,,,,,,00000000001.20.6], shards=11 2026-03-20T11:45:37.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.622+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196bb270:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.622+0000 7f8f59593900 10 trimming bilog shard 10 of :dipepu-37[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42]) at marker 00000000001.20.6 2026-03-20T11:45:37.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.622+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.622+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e1967fe70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.622+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e1967fe70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.622+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42.10 2026-03-20T11:45:37.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.622+0000 7f8f59593900 20 run: stack=0x557e196baa80 is io blocked 2026-03-20T11:45:37.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.623+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e1967fe70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.623+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e1967fe70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.623+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e1967fe70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.623+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e1967fe70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.623+0000 7f8f59593900 15 stack 0x557e196baa80 end 2026-03-20T11:45:37.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.623+0000 7f8f59593900 20 run: stack=0x557e196baa80 is done 2026-03-20T11:45:37.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.623+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196bb270:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.623+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.623+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:45:37.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.623+0000 7f8f59593900 15 stack 0x557e196ad300 end 2026-03-20T11:45:37.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.623+0000 7f8f59593900 20 run: stack=0x557e196ad300 is done 2026-03-20T11:45:37.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.623+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.623+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.623+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.623+0000 7f8f59593900 4 starting trim on bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:45:37.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.623+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196fee60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.623+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196fee60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.623+0000 7f8f59593900 20 enqueued request req=0x557e19671360 2026-03-20T11:45:37.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.623+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.623+0000 7f8f59593900 20 req: 0x557e19671360 2026-03-20T11:45:37.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.623+0000 7f8f59593900 20 run: stack=0x557e196ad300 is io blocked 2026-03-20T11:45:37.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.623+0000 7f8f057ea640 20 dequeued request req=0x557e19671360 2026-03-20T11:45:37.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.623+0000 7f8f057ea640 20 RGWWQ: empty 2026-03-20T11:45:37.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.623+0000 7f8f057ea640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 : miss 2026-03-20T11:45:37.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.623+0000 7f8f057ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f057ea640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f057ea640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 info.flags=0x17 2026-03-20T11:45:37.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f057ea640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 to cache LRU end 2026-03-20T11:45:37.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f057ea640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f057ea640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f057ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:45:37.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f057ea640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-59 : miss 2026-03-20T11:45:37.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f057ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f057ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f057ea640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-59 info.flags=0x0 2026-03-20T11:45:37.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f057ea640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-59 to cache LRU end 2026-03-20T11:45:37.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f057ea640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-59 : miss 2026-03-20T11:45:37.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f057ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f057ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f057ea640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-59 info.flags=0x0 2026-03-20T11:45:37.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f057ea640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-59 to cache LRU end 2026-03-20T11:45:37.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f057ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19): adding source pipe: {s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f057ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19): adding dest pipe: {s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f057ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f057ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f057ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:45:37.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196fee60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196fee60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196fee60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196fee60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196bb270:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e19683e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e19683e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = a1aa8bd615cbe264b96110d73d24214cbd48efdd6ce83f66b6fbf9e636835f50 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:a1aa8bd615cbe264b96110d73d24214cbd48efdd6ce83f66b6fbf9e636835f50 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 9d0df5654f08afafab34c7f0c7e127cc3f1ed9ac5818fa340bc5d15289671eb0 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=9d0df5654f08afafab34c7f0c7e127cc3f1ed9ac5818fa340bc5d15289671eb0 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=28, curl_handle=0x557e196861c0 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8eb2ffd640 20 link_request req_data=0x557e196868d0 req_data->id=28, curl_handle=0x557e196861c0 2026-03-20T11:45:37.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.624+0000 7f8f59593900 20 run: stack=0x557e196baa80 is io blocked 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196b95b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196b95b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d3310:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f59593900 15 stack 0x557e196be570 end 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f59593900 20 run: stack=0x557e196be570 is done 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f59593900 20 collect(): s=0x557e1957ee80 stack=0x557e196be570 is complete 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f59593900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f59593900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f59593900 15 stack 0x557e1957ee80 end 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f59593900 20 stack->operate() returned ret=-22 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is done 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f59593900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f59593900 4 starting trim on bucket=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196fee60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196fee60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f59593900 20 enqueued request req=0x557e196bf240 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f59593900 20 req: 0x557e196bf240 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is io blocked 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f04fe9640 20 dequeued request req=0x557e196bf240 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f04fe9640 20 RGWWQ: empty 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f04fe9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 : miss 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f04fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f04fe9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=386 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f04fe9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 info.flags=0x17 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f04fe9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 to cache LRU end 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f04fe9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f04fe9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f04fe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 2026-03-20T11:45:37.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f04fe9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-6 : miss 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f04fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f04fe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f04fe9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-6 info.flags=0x0 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f04fe9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-6 to cache LRU end 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f04fe9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-6 : miss 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f04fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f04fe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f04fe9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-6 info.flags=0x0 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f04fe9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-6 to cache LRU end 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f04fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11): adding source pipe: {s={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f04fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11): adding dest pipe: {s={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f04fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f04fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.626+0000 7f8f04fe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196fee60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196fee60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196fee60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196fee60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d3960:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d8e00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d8e00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = cdee89c8e75db7fb846ba1eeeddb2bd1a13771a835f6f83693cb3264f90a68ff 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:cdee89c8e75db7fb846ba1eeeddb2bd1a13771a835f6f83693cb3264f90a68ff 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 06598c962d944773297097052116f5675955eaaaae36ca9a9f1886af9f7a2d8a 2026-03-20T11:45:37.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=06598c962d944773297097052116f5675955eaaaae36ca9a9f1886af9f7a2d8a 2026-03-20T11:45:37.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=29, curl_handle=0x557e19681800 2026-03-20T11:45:37.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8eb2ffd640 20 link_request req_data=0x557e196823d0 req_data->id=29, curl_handle=0x557e19681800 2026-03-20T11:45:37.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.627+0000 7f8f59593900 20 run: stack=0x557e196be570 is io blocked 2026-03-20T11:45:37.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196da060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196da060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e19684590:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e19684590:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196da060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196b4490:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 15 stack 0x557e196c3030 end 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 run: stack=0x557e196c3030 is done 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196da060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 collect(): s=0x557e196700a0 stack=0x557e196c3030 is complete 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 15 stack 0x557e196700a0 end 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 stack->operate() returned ret=-22 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 run: stack=0x557e196700a0 is done 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196c2540:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 15 stack 0x557e196cf7c0 end 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 run: stack=0x557e196cf7c0 is done 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 collect(): s=0x557e1967dc20 stack=0x557e196cf7c0 is complete 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) markers=[,00000000002.6.6,,00000000001.10.6,,00000000001.6.6,00000000002.6.6,00000000001.6.6,00000000001.12.6,00000000001.4.6,], shards=11 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 4 starting trim on bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e196d6840:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 10 trimming bilog shard 1 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000002.6.6 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 10 trimming bilog shard 3 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.10.6 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 10 trimming bilog shard 5 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.6.6 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 10 trimming bilog shard 6 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000002.6.6 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 10 trimming bilog shard 7 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.6.6 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 10 trimming bilog shard 8 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.12.6 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 10 trimming bilog shard 9 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.4.6 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196d44b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196c8c60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1920d8f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d7fa0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196d4e50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e1920d030:op=0x557e196d29b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e196c8a50:op=0x557e192121d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 enqueued request req=0x557e196cd050 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 req: 0x557e196cd050 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 run: stack=0x557e196700a0 is io blocked 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196d44b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.1 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f20ff9640 20 dequeued request req=0x557e196cd050 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f20ff9640 20 RGWWQ: empty 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f20ff9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 : miss 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 run: stack=0x557e196cf7c0 is io blocked 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196c8c60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.3 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f20ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 run: stack=0x557e196c3030 is io blocked 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1920d8f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.5 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 run: stack=0x557e19688770 is io blocked 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d7fa0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.6 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is io blocked 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196d4e50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.7 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 run: stack=0x557e196bcde0 is io blocked 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e1920d030:op=0x557e196d29b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.8 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 run: stack=0x557e1920d030 is io blocked 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 cr:s=0x557e196c8a50:op=0x557e192121d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.9 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.629+0000 7f8f59593900 20 run: stack=0x557e196c8a50 is io blocked 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.630+0000 7f8f20ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.630+0000 7f8f20ff9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 info.flags=0x17 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.630+0000 7f8f20ff9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 to cache LRU end 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.630+0000 7f8f20ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.630+0000 7f8f20ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.630+0000 7f8f20ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:45:37.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.630+0000 7f8f20ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-21 : miss 2026-03-20T11:45:37.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.630+0000 7f8f20ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.630+0000 7f8f20ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.630+0000 7f8f20ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-21 info.flags=0x0 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.630+0000 7f8f20ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-21 to cache LRU end 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.630+0000 7f8f20ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-21 : miss 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.630+0000 7f8f20ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f20ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196d44b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f20ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-21 info.flags=0x0 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f20ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-21 to cache LRU end 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196d44b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196c8c60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196d44b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196c8c60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196d44b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 15 stack 0x557e196cf7c0 end 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f20ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26): adding source pipe: {s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 run: stack=0x557e196cf7c0 is done 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196c8c60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e196d6840:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196c8c60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 15 stack 0x557e196c3030 end 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 run: stack=0x557e196c3030 is done 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e196d6840:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f20ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26): adding dest pipe: {s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f20ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f20ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f20ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196c44d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 run: stack=0x557e196700a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196b4490:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196b4490:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 75e289abf9d3ef963404d62d511bf08caedcfac06980598f13b35c4330c90726 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:75e289abf9d3ef963404d62d511bf08caedcfac06980598f13b35c4330c90726 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = f3c4c6898e09cd4b254785679c7bf086e0262172b25f4af665c3a2ccc15ea8ff 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=f3c4c6898e09cd4b254785679c7bf086e0262172b25f4af665c3a2ccc15ea8ff 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=30, curl_handle=0x557e196b6260 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8eb2ffd640 20 link_request req_data=0x557e196b6970 req_data->id=30, curl_handle=0x557e196b6260 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 run: stack=0x557e196c3030 is io blocked 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e1920d030:op=0x557e196d29b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e1920d030:op=0x557e196d29b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e1920d030:op=0x557e196d29b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e1920d030:op=0x557e196d29b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 15 stack 0x557e1920d030 end 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 run: stack=0x557e1920d030 is done 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e196d6840:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e196c8a50:op=0x557e192121d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e196c8a50:op=0x557e192121d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1920d8f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e196c8a50:op=0x557e192121d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1920d8f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e196c8a50:op=0x557e192121d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 15 stack 0x557e196c8a50 end 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 run: stack=0x557e196c8a50 is done 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1920d8f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e196d6840:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1920d8f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 15 stack 0x557e19688770 end 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 run: stack=0x557e19688770 is done 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e196d6840:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.631+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196d4e50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196d4e50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196d4e50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196d4e50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 15 stack 0x557e196bcde0 end 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 20 run: stack=0x557e196bcde0 is done 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e196d6840:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d7fa0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d7fa0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d7fa0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d7fa0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 15 stack 0x557e196c7af0 end 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is done 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e196d6840:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 15 stack 0x557e1967dc20 end 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is done 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 4 starting trim on bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1920d180:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1920d180:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 20 enqueued request req=0x557e1957ec90 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 20 req: 0x557e1957ec90 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is io blocked 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f047e8640 20 dequeued request req=0x557e1957ec90 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f047e8640 20 RGWWQ: empty 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f047e8640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 : miss 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f047e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f047e8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f047e8640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 info.flags=0x17 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f047e8640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 to cache LRU end 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f047e8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f047e8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f047e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f047e8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-41 : miss 2026-03-20T11:45:37.633 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.632+0000 7f8f047e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.633 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:37.634+0000 7fd0b43b6640 -1 req 6409662300012938140 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f047e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f047e8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-41 info.flags=0x0 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f047e8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-41 to cache LRU end 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f047e8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-41 : miss 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f047e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f047e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f047e8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-41 info.flags=0x0 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f047e8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-41 to cache LRU end 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f047e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1): adding source pipe: {s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f047e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1): adding dest pipe: {s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f047e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f047e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f047e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1920d180:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1920d180:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1920d180:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1920d180:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196da060:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196b95b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196b95b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 01d83f6d2db2246be4b33ed2b5b68f2192d81455295666ca07e0d5e989f2a27a 2026-03-20T11:45:37.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.635 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.635 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.635 INFO:teuthology.orchestra.run.vm02.stderr:01d83f6d2db2246be4b33ed2b5b68f2192d81455295666ca07e0d5e989f2a27a 2026-03-20T11:45:37.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 0d47714231d07b5d4a318bcc6e17643da688425bdae8991d2a7b01bba3be083d 2026-03-20T11:45:37.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=0d47714231d07b5d4a318bcc6e17643da688425bdae8991d2a7b01bba3be083d 2026-03-20T11:45:37.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=31, curl_handle=0x557e196b2160 2026-03-20T11:45:37.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.633+0000 7f8eb2ffd640 20 link_request req_data=0x557e196b2870 req_data->id=31, curl_handle=0x557e196b2160 2026-03-20T11:45:37.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.634+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is io blocked 2026-03-20T11:45:37.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e19683e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e19683e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e19683e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e19683e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196bb270:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 15 stack 0x557e196baa80 end 2026-03-20T11:45:37.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 20 run: stack=0x557e196baa80 is done 2026-03-20T11:45:37.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 20 collect(): s=0x557e196ad300 stack=0x557e196baa80 is complete 2026-03-20T11:45:37.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) markers=[,00000000001.44.6,,,,00000000001.24.6,,,00000000001.44.6,,00000000001.39.6], shards=11 2026-03-20T11:45:37.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196d6840:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 10 trimming bilog shard 1 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.44.6 2026-03-20T11:45:37.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 10 trimming bilog shard 5 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.24.6 2026-03-20T11:45:37.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 10 trimming bilog shard 8 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.44.6 2026-03-20T11:45:37.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 10 trimming bilog shard 10 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.39.6 2026-03-20T11:45:37.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e1967fe70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196d2940:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1920bed0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e19210f00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e1967fe70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.1 2026-03-20T11:45:37.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 20 run: stack=0x557e196baa80 is io blocked 2026-03-20T11:45:37.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196d2940:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.5 2026-03-20T11:45:37.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 20 run: stack=0x557e196bcde0 is io blocked 2026-03-20T11:45:37.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1920bed0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.8 2026-03-20T11:45:37.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 20 run: stack=0x557e19688770 is io blocked 2026-03-20T11:45:37.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e19210f00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.10 2026-03-20T11:45:37.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.667+0000 7f8f59593900 20 run: stack=0x557e196cf7c0 is io blocked 2026-03-20T11:45:37.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.668+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e1967fe70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.668+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e1967fe70:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:37.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.668+0000 7f8f59593900 15 stack 0x557e196baa80 end 2026-03-20T11:45:37.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.668+0000 7f8f59593900 20 stack->operate() returned ret=-61 2026-03-20T11:45:37.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.668+0000 7f8f59593900 20 run: stack=0x557e196baa80 is done 2026-03-20T11:45:37.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.668+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196d6840:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.668+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1920bed0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1920bed0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 15 stack 0x557e19688770 end 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 stack->operate() returned ret=-61 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 run: stack=0x557e19688770 is done 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196d6840:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e19210f00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e19210f00:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 15 stack 0x557e196cf7c0 end 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 stack->operate() returned ret=-61 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 run: stack=0x557e196cf7c0 is done 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d8e00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196d6840:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d8e00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196d2940:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196d2940:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 15 stack 0x557e196bcde0 end 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 stack->operate() returned ret=-61 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 run: stack=0x557e196bcde0 is done 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d8e00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196d6840:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d8e00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 15 stack 0x557e196ad300 end 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 run: stack=0x557e196ad300 is done 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d3960:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 15 stack 0x557e196be570 end 2026-03-20T11:45:37.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 run: stack=0x557e196be570 is done 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 collect(): s=0x557e1957ee80 stack=0x557e196be570 is complete 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 10 min_generation is 1 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 4 starting trim on bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196bb270:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11.0 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11.1 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11.2 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11.3 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11.4 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11.5 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11.6 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11.7 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11.8 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11.9 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11.10 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920b080:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e19683e10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196cffb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196ff680:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e1920a900:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e19684590:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196d2940:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1967fe70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196c6330:op=0x557e192c61f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196bc200:op=0x557e1920ffc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196bc340:op=0x557e19210ed0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 enqueued request req=0x557e1920e8a0 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 req: 0x557e1920e8a0 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 run: stack=0x557e196ad300 is io blocked 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920b080:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 run: stack=0x557e196be570 is io blocked 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e19683e10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f167fc640 20 dequeued request req=0x557e1920e8a0 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f167fc640 20 RGWWQ: empty 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 run: stack=0x557e196bcde0 is io blocked 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196cffb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f167fc640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 : miss 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f167fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 run: stack=0x557e196cf7c0 is io blocked 2026-03-20T11:45:37.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196ff680:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 run: stack=0x557e19688770 is io blocked 2026-03-20T11:45:37.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e1920a900:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 run: stack=0x557e196baa80 is io blocked 2026-03-20T11:45:37.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e19684590:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 run: stack=0x557e196d3220 is io blocked 2026-03-20T11:45:37.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196d2940:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 run: stack=0x557e196d8570 is io blocked 2026-03-20T11:45:37.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1967fe70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 run: stack=0x557e19701570 is io blocked 2026-03-20T11:45:37.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196c6330:op=0x557e192c61f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 run: stack=0x557e196c6330 is io blocked 2026-03-20T11:45:37.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.669+0000 7f8f59593900 20 cr:s=0x557e196bc200:op=0x557e1920ffc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.670+0000 7f8f59593900 20 run: stack=0x557e196bc200 is io blocked 2026-03-20T11:45:37.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.670+0000 7f8f59593900 20 cr:s=0x557e196bc340:op=0x557e19210ed0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.670+0000 7f8f59593900 20 run: stack=0x557e196bc340 is io blocked 2026-03-20T11:45:37.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.670+0000 7f8f167fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:37.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.670+0000 7f8f167fc640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 info.flags=0x17 2026-03-20T11:45:37.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.670+0000 7f8f167fc640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 to cache LRU end 2026-03-20T11:45:37.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.670+0000 7f8f167fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.670+0000 7f8f167fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.670+0000 7f8f167fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:45:37.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.670+0000 7f8f167fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-49 : miss 2026-03-20T11:45:37.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.670+0000 7f8f167fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f167fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f167fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-49 info.flags=0x0 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f167fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-49 to cache LRU end 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f167fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-49 : miss 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f167fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e1920a900:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e1920a900:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e1920a900:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e1920a900:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f59593900 15 stack 0x557e196baa80 end 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f59593900 20 run: stack=0x557e196baa80 is done 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196bb270:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f167fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f167fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-49 info.flags=0x0 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f167fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-49 to cache LRU end 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f167fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9): adding source pipe: {s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e19684590:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e19684590:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920b080:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1967fe70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196cffb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196d2940:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e19684590:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920b080:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f167fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9): adding dest pipe: {s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1967fe70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f167fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e19683e10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196cffb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196d2940:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e19684590:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f59593900 15 stack 0x557e196d3220 end 2026-03-20T11:45:37.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f59593900 20 run: stack=0x557e196d3220 is done 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920b080:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1967fe70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e19683e10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.671+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196cffb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196d2940:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196bb270:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920b080:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 15 stack 0x557e196be570 end 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 run: stack=0x557e196be570 is done 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1967fe70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 15 stack 0x557e19701570 end 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 run: stack=0x557e19701570 is done 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e19683e10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196cffb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 15 stack 0x557e196cf7c0 end 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 run: stack=0x557e196cf7c0 is done 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196d2940:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 15 stack 0x557e196d8570 end 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 run: stack=0x557e196d8570 is done 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196bb270:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e19683e10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 15 stack 0x557e196bcde0 end 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 run: stack=0x557e196bcde0 is done 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196bb270:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196bb270:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196bb270:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196bb270:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f167fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f167fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1920a900:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e196d3e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e196d3e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.672 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.672 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.672 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.672 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.672 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.672 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.672 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.672 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.672 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = db0842631e371daf92064c928173a00c37e310efcdf819b8516db9737894824c 2026-03-20T11:45:37.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.672 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.672 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.672 INFO:teuthology.orchestra.run.vm02.stderr:db0842631e371daf92064c928173a00c37e310efcdf819b8516db9737894824c 2026-03-20T11:45:37.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = c8b43401b5aceeec801d8f34721a9b1addea59683056d306932ec5e7a5fd81ba 2026-03-20T11:45:37.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c8b43401b5aceeec801d8f34721a9b1addea59683056d306932ec5e7a5fd81ba 2026-03-20T11:45:37.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=32, curl_handle=0x557e19681800 2026-03-20T11:45:37.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8eb2ffd640 20 link_request req_data=0x557e196823d0 req_data->id=32, curl_handle=0x557e19681800 2026-03-20T11:45:37.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 run: stack=0x557e19701570 is io blocked 2026-03-20T11:45:37.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196ff680:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196ff680:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196ff680:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196ff680:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 15 stack 0x557e19688770 end 2026-03-20T11:45:37.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 run: stack=0x557e19688770 is done 2026-03-20T11:45:37.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196bb270:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e196bc340:op=0x557e19210ed0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e196bc340:op=0x557e19210ed0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e196bc340:op=0x557e19210ed0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e196bc340:op=0x557e19210ed0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 15 stack 0x557e196bc340 end 2026-03-20T11:45:37.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 run: stack=0x557e196bc340 is done 2026-03-20T11:45:37.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196bb270:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e196c6330:op=0x557e192c61f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e196c6330:op=0x557e192c61f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e196bc200:op=0x557e1920ffc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e196c6330:op=0x557e192c61f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e196bc200:op=0x557e1920ffc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e196c6330:op=0x557e192c61f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 15 stack 0x557e196c6330 end 2026-03-20T11:45:37.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 run: stack=0x557e196c6330 is done 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e196bc200:op=0x557e1920ffc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196bb270:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e196bc200:op=0x557e1920ffc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 15 stack 0x557e196bc200 end 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 run: stack=0x557e196bc200 is done 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196bb270:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196ff680:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196ff680:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 enqueued request req=0x557e196c5e90 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 req: 0x557e196c5e90 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is io blocked 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f177fe640 20 dequeued request req=0x557e196c5e90 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.672+0000 7f8f177fe640 20 RGWWQ: empty 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.674+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196b4490:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.674+0000 7f8f177fe640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 info.flags=0x11 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.674+0000 7f8f177fe640 10 rgw async rados processor: moving a1.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 to cache LRU end 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.674+0000 7f8f177fe640 10 rgw async rados processor: distributing notification oid=a1.rgw.control:notify.5 cni=[op: 0, obj: a1.rgw.meta:root:.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11, ofs0, ns] 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.674+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196b4490:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.674+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196b4490:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.674+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196b4490:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.674+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196c44d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.674+0000 7f8f59593900 15 stack 0x557e196c3030 end 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.674+0000 7f8f59593900 20 run: stack=0x557e196c3030 is done 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.674+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.674+0000 7f8f59593900 20 collect(): s=0x557e196700a0 stack=0x557e196c3030 is complete 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.674+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.674+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-21[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26]) markers=[,,,,,,,,00000000001.60.14,,], shards=11 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.674+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196cffb0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.674+0000 7f8f59593900 10 trimming bilog shard 8 of :dipepu-21[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26]) at marker 00000000001.60.14 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.674+0000 7f8f59593900 20 run: stack=0x557e196700a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.674+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196d44b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.674+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196d44b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.674+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26.8 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.674+0000 7f8f59593900 20 run: stack=0x557e19688770 is io blocked 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.674+0000 7f8f23fff640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 335007449317 cookie 94000080666336 notifier 6226 bl.length()=565 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.674+0000 7f8f23fff640 10 rgw watcher librados: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 info.flags=0x11 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.674+0000 7f8f23fff640 10 rgw watcher librados: moving a1.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 to cache LRU end 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196d44b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196d44b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196d44b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196d44b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f59593900 15 stack 0x557e19688770 end 2026-03-20T11:45:37.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f59593900 20 run: stack=0x557e19688770 is done 2026-03-20T11:45:37.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196cffb0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:45:37.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f59593900 15 stack 0x557e196700a0 end 2026-03-20T11:45:37.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f59593900 20 run: stack=0x557e196700a0 is done 2026-03-20T11:45:37.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f59593900 4 starting trim on bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:45:37.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f59593900 20 enqueued request req=0x557e19671360 2026-03-20T11:45:37.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f59593900 20 req: 0x557e19671360 2026-03-20T11:45:37.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f16ffd640 20 dequeued request req=0x557e19671360 2026-03-20T11:45:37.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f16ffd640 20 RGWWQ: empty 2026-03-20T11:45:37.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f16ffd640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 : miss 2026-03-20T11:45:37.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f16ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f59593900 20 run: stack=0x557e19688770 is io blocked 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f16ffd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f16ffd640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 info.flags=0x17 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f16ffd640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 to cache LRU end 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f16ffd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f16ffd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f16ffd640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.675+0000 7f8f16ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f16ffd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-45 : miss 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f16ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f177fe640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-6[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11]): orig_sources= new_sources= 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f177fe640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-6[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11]): potential sources added=[] removed=[] 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f177fe640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-6[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11]): orig_dests= new_dests= 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f177fe640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-6[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11]): potential dests added=[] removed=[] 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196ff680:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196ff680:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196b95b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196b95b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196ff680:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196da060:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 15 stack 0x557e196c7af0 end 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is done 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196ff680:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 collect(): s=0x557e1967dc20 stack=0x557e196c7af0 is complete 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 cr:s=0x557e1967dc20:op=0x557e1967b090:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 15 stack 0x557e1967dc20 end 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 stack->operate() returned ret=-22 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 run: stack=0x557e1967dc20 is done 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196c44d0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 4 starting trim on bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196c44d0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 enqueued request req=0x557e191a8f40 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 req: 0x557e191a8f40 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is io blocked 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1920af20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.677 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1920af20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 enqueued request req=0x557e1920bb30 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 req: 0x557e191a8f40 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 req: 0x557e1920bb30 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is io blocked 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f17fff640 20 dequeued request req=0x557e191a8f40 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f17fff640 20 RGWWQ: 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f17fff640 20 req: 0x557e1920bb30 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f15ffb640 20 dequeued request req=0x557e1920bb30 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f15ffb640 20 RGWWQ: empty 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f15ffb640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 : miss 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f15ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f16ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f16ffd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-45 info.flags=0x0 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f16ffd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-45 to cache LRU end 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f16ffd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-45 : miss 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f16ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f16ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f16ffd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-45 info.flags=0x0 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f16ffd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-45 to cache LRU end 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f16ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5): adding source pipe: {s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f16ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5): adding dest pipe: {s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f16ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f16ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f16ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196c4c90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 run: stack=0x557e19688770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196d8e00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196d8e00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f15ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f15ffb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 info.flags=0x17 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f15ffb640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 to cache LRU end 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f15ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f15ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f15ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f15ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f15ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-33 : miss 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f15ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.678 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = b8a44efa41099afdc8ca7e0191331f9f26514988995ac759e981cdd694287325 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:b8a44efa41099afdc8ca7e0191331f9f26514988995ac759e981cdd694287325 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.676+0000 7f8f15ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f15ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-33 info.flags=0x0 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f15ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-33 to cache LRU end 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f15ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-33 : miss 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f15ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 012714a0e5b4a9e79cd42e35e270e7935281b3dd05658d3f607c8ca7d67d3b0d 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=012714a0e5b4a9e79cd42e35e270e7935281b3dd05658d3f607c8ca7d67d3b0d 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=33, curl_handle=0x557e196b2160 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8eb2ffd640 20 link_request req_data=0x557e196b2870 req_data->id=33, curl_handle=0x557e196b2160 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 run: stack=0x557e196d8570 is io blocked 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f17fff640 10 rgw async rados processor: removing a1.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 from cache 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f17fff640 10 rgw async rados processor: distributing notification oid=a1.rgw.control:notify.5 cni=[op: 1, obj: a1.rgw.meta:root:.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11, ofs0, ns] 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f15ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f15ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-33 info.flags=0x0 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f15ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-33 to cache LRU end 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f15ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38): adding source pipe: {s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f15ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38): adding dest pipe: {s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f15ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f15ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f15ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1920af20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1920af20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1920af20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1920af20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196da040:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196cffb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196cffb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f23fff640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 335007449318 cookie 94000080666336 notifier 6226 bl.length()=205 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 14f598957e36febbb78c797cf0c94512061d40170ab2f37b830e49088b60828b 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:14f598957e36febbb78c797cf0c94512061d40170ab2f37b830e49088b60828b 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 608a9b182a4b82598c61f335e1a6d74b556e30f67cc7f1a6e7bab8527a5653a5 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=608a9b182a4b82598c61f335e1a6d74b556e30f67cc7f1a6e7bab8527a5653a5 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=34, curl_handle=0x557e196b6260 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8eb2ffd640 20 link_request req_data=0x557e196b6970 req_data->id=34, curl_handle=0x557e196b6260 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 run: stack=0x557e196cf7c0 is io blocked 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196c44d0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196c44d0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196c44d0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196c44d0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 15 stack 0x557e1957ee80 end 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is done 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 4 starting trim on bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 enqueued request req=0x557e196c3c00 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 req: 0x557e196c3c00 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is io blocked 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f157fa640 20 dequeued request req=0x557e196c3c00 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f157fa640 20 RGWWQ: empty 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f157fa640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 : miss 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.677+0000 7f8f157fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f157fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f157fa640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 info.flags=0x17 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f157fa640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 to cache LRU end 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f157fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f157fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f157fa640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f157fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f157fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-47 : miss 2026-03-20T11:45:37.680 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f157fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f157fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f157fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-47 info.flags=0x0 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f157fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-47 to cache LRU end 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f157fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-47 : miss 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f157fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f157fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f157fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-47 info.flags=0x0 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f157fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-47 to cache LRU end 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f157fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7): adding source pipe: {s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f157fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7): adding dest pipe: {s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f157fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f157fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f157fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e19684590:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196b95b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196b95b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = b2bf45f9aefb1c6edb443dcee728c81c2371ffae8a25862ec960ca7b69a57296 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:b2bf45f9aefb1c6edb443dcee728c81c2371ffae8a25862ec960ca7b69a57296 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 46a0098adde9a1e0e36189ec103e34258fb31f891b23a33ddc2f4dd2e05097a0 2026-03-20T11:45:37.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=46a0098adde9a1e0e36189ec103e34258fb31f891b23a33ddc2f4dd2e05097a0 2026-03-20T11:45:37.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=35, curl_handle=0x557e196861c0 2026-03-20T11:45:37.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8eb2ffd640 20 link_request req_data=0x7f8ee000f390 req_data->id=35, curl_handle=0x557e196861c0 2026-03-20T11:45:37.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.678+0000 7f8f59593900 20 run: stack=0x557e196bcde0 is io blocked 2026-03-20T11:45:37.682 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:37.677+0000 7fcfcb1e4640 -1 req 698565243537675061 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:37.682 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:37.678+0000 7fcfc29d3640 -1 req 7019937883558623654 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:37.682 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:37.679+0000 7fcff9a41640 -1 req 1271669131849986113 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:37.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.686+0000 7f8f027e4640 20 rgw rados thread: process(): notifying mdlog change, shard_id=1 2026-03-20T11:45:37.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.686+0000 7f8f027e4640 20 rgw rados thread: cr:s=0x7f8f480e9960:op=0x7f8f480de6b0:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:45:37.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.686+0000 7f8f027e4640 20 rgw rados thread: cr:s=0x7f8f480e9960:op=0x7f8f480de6b0:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:45:37.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.686+0000 7f8f027e4640 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.686+0000 7f8f027e4640 20 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.686+0000 7f8f027e4640 20 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.686+0000 7f8f027e4640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.684 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.684 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:080a9ed428559ef602668b4c00f114f1a11c3f6b02a435f0bdc154578e4d7f22 2026-03-20T11:45:37.684 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.684 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.686+0000 7f8f027e4640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]payload request hash = 080a9ed428559ef602668b4c00f114f1a11c3f6b02a435f0bdc154578e4d7f22 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.686+0000 7f8f027e4640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]canonical request = POST 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:notify=&rgwx-zonegroup=a&type=metadata 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:080a9ed428559ef602668b4c00f114f1a11c3f6b02a435f0bdc154578e4d7f22 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:080a9ed428559ef602668b4c00f114f1a11c3f6b02a435f0bdc154578e4d7f22 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.686+0000 7f8f027e4640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]canonical request hash = 10f7836e079329bb817eeb6033ee47060ef2413f2afe2d7ec65b90dd588b0d0e 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.686+0000 7f8f027e4640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:10f7836e079329bb817eeb6033ee47060ef2413f2afe2d7ec65b90dd588b0d0e 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.686+0000 7f8f027e4640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.686+0000 7f8f027e4640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.686+0000 7f8f027e4640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.686+0000 7f8f027e4640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.686+0000 7f8f027e4640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]generated signature = abd415ddd88a44bdcfaca57fa96283cbdaa90ec41dae951b09c2443607e342db 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.686+0000 7f8f027e4640 20 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=abd415ddd88a44bdcfaca57fa96283cbdaa90ec41dae951b09c2443607e342db 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.686+0000 7f8f027e4640 20 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: 080a9ed428559ef602668b4c00f114f1a11c3f6b02a435f0bdc154578e4d7f22 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.686+0000 7f8f027e4640 20 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.686+0000 7f8f027e4640 20 sending request to http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.686+0000 7f8f027e4640 20 register_request mgr=0x557e192008f8 req_data->id=0, curl_handle=0x7f8f48098c10 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.686+0000 7f8f027e4640 20 run: stack=0x7f8f480e9960 is io blocked 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.686+0000 7f8f02fe5640 20 link_request req_data=0x7f8f480bb490 req_data->id=0, curl_handle=0x7f8f48098c10 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.687+0000 7f8f027e4640 20 rgw rados thread: cr:s=0x7f8f480e9960:op=0x7f8f480de6b0:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.687+0000 7f8f027e4640 20 rgw rados thread: cr:s=0x7f8f480e9960:op=0x7f8f480de6b0:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.687+0000 7f8f027e4640 20 rgw rados thread: cr:s=0x7f8f480e9960:op=0x7f8f480de6b0:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.687+0000 7f8f027e4640 20 rgw rados thread: cr:s=0x7f8f480e9960:op=0x7f8f480de6b0:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.687+0000 7f8f027e4640 15 stack 0x7f8f480e9960 end 2026-03-20T11:45:37.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.687+0000 7f8f027e4640 20 run: stack=0x7f8f480e9960 is done 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e196d3e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e196d3e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e196d3e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e196d3e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1920a900:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f59593900 15 stack 0x557e19701570 end 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f59593900 20 run: stack=0x557e19701570 is done 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f59593900 20 collect(): s=0x557e196ad300 stack=0x557e19701570 is complete 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f59593900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f59593900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e1966e820:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f59593900 15 stack 0x557e196ad300 end 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f59593900 20 stack->operate() returned ret=-22 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f59593900 20 run: stack=0x557e196ad300 is done 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f59593900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19210960:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f59593900 4 starting trim on bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f59593900 20 enqueued request req=0x557e196d2f30 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f59593900 20 req: 0x557e196d2f30 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f59593900 20 run: stack=0x557e196ad300 is io blocked 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f14ff9640 20 dequeued request req=0x557e196d2f30 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f14ff9640 20 RGWWQ: empty 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f14ff9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 : miss 2026-03-20T11:45:37.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f14ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f14ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f14ff9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 info.flags=0x17 2026-03-20T11:45:37.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f14ff9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 to cache LRU end 2026-03-20T11:45:37.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f14ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f14ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f14ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:45:37.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f14ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-58 : miss 2026-03-20T11:45:37.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.713+0000 7f8f14ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f14ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f14ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-58 info.flags=0x0 2026-03-20T11:45:37.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f14ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-58 to cache LRU end 2026-03-20T11:45:37.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f14ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-58 : miss 2026-03-20T11:45:37.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f14ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f14ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f14ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-58 info.flags=0x0 2026-03-20T11:45:37.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f14ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-58 to cache LRU end 2026-03-20T11:45:37.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f14ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18): adding source pipe: {s={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f14ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18): adding dest pipe: {s={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f14ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f14ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f14ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:45:37.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19210960:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1920feb0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19210960:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e196c44d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e196c44d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 8218cc6dad7e9dfdd034543c142b70aaa99e561fc402e7191ea2556ad969caf0 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:8218cc6dad7e9dfdd034543c142b70aaa99e561fc402e7191ea2556ad969caf0 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 9c44c1e3f6acbe7358474975c89c42f40729bca73d603e3c1217b8802eeb64f5 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=9c44c1e3f6acbe7358474975c89c42f40729bca73d603e3c1217b8802eeb64f5 2026-03-20T11:45:37.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=36, curl_handle=0x557e19681800 2026-03-20T11:45:37.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8eb2ffd640 20 link_request req_data=0x557e196823d0 req_data->id=36, curl_handle=0x557e19681800 2026-03-20T11:45:37.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.714+0000 7f8f59593900 20 run: stack=0x557e19701570 is io blocked 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196d8e00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196d8e00:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196c4c90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f59593900 15 stack 0x557e196d8570 end 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f59593900 20 run: stack=0x557e196d8570 is done 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f59593900 20 collect(): s=0x557e19688770 stack=0x557e196d8570 is complete 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f59593900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f59593900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1967c360:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f59593900 15 stack 0x557e19688770 end 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f59593900 20 stack->operate() returned ret=-22 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f59593900 20 run: stack=0x557e19688770 is done 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f59593900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f59593900 4 starting trim on bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f59593900 20 enqueued request req=0x557e19671360 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f59593900 20 req: 0x557e19671360 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f59593900 20 run: stack=0x557e19688770 is io blocked 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f0f7fe640 20 dequeued request req=0x557e19671360 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f0f7fe640 20 RGWWQ: empty 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f0f7fe640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 : miss 2026-03-20T11:45:37.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f0f7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f0f7fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f0f7fe640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 info.flags=0x17 2026-03-20T11:45:37.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f0f7fe640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 to cache LRU end 2026-03-20T11:45:37.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f0f7fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f0f7fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:45:37.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f0f7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:45:37.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f0f7fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-30 : miss 2026-03-20T11:45:37.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f0f7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f0f7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f0f7fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-30 info.flags=0x0 2026-03-20T11:45:37.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f0f7fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-30 to cache LRU end 2026-03-20T11:45:37.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f0f7fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-30 : miss 2026-03-20T11:45:37.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.719+0000 7f8f0f7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196cffb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196cffb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:37.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196da040:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 15 stack 0x557e196cf7c0 end 2026-03-20T11:45:37.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 run: stack=0x557e196cf7c0 is done 2026-03-20T11:45:37.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 collect(): s=0x557e196c7af0 stack=0x557e196cf7c0 is complete 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 15 stack 0x557e196c7af0 end 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 stack->operate() returned ret=-22 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is done 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 4 starting trim on bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d8e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d8e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 enqueued request req=0x557e1920bb30 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 req: 0x557e1920bb30 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is io blocked 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f0ffff640 20 dequeued request req=0x557e1920bb30 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f0ffff640 20 RGWWQ: empty 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f0f7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f0f7fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-30 info.flags=0x0 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f0f7fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-30 to cache LRU end 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f0ffff640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 : miss 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f0f7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35): adding source pipe: {s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f0ffff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f0f7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35): adding dest pipe: {s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f0f7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f0f7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f0f7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196da040:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 run: stack=0x557e19688770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196cffb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196cffb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.719 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 5df27fe8184a13a871f14b008474d71a94c153f198f19fda2a9ccbed760bdec9 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:5df27fe8184a13a871f14b008474d71a94c153f198f19fda2a9ccbed760bdec9 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = d97d5f52c981e61037db9753b996a675890138d99835bcf5bd1a73c6f61a454c 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=d97d5f52c981e61037db9753b996a675890138d99835bcf5bd1a73c6f61a454c 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=37, curl_handle=0x557e196b6260 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8eb2ffd640 20 link_request req_data=0x557e196b6970 req_data->id=37, curl_handle=0x557e196b6260 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f59593900 20 run: stack=0x557e196cf7c0 is io blocked 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f0ffff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f0ffff640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 info.flags=0x17 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f0ffff640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 to cache LRU end 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f0ffff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f0ffff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f0ffff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f0ffff640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-20 : miss 2026-03-20T11:45:37.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f0ffff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.720 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:37.721+0000 7fcfca1e2640 -1 req 6132850080019405008 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f0ffff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f0ffff640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-20 info.flags=0x0 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f0ffff640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-20 to cache LRU end 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f0ffff640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-20 : miss 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.720+0000 7f8f0ffff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f0ffff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f0ffff640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-20 info.flags=0x0 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f0ffff640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-20 to cache LRU end 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f0ffff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25): adding source pipe: {s={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f0ffff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25): adding dest pipe: {s={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f0ffff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f0ffff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f0ffff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d8e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d8e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196b95b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196b95b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d8e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e19684590:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 15 stack 0x557e196bcde0 end 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 run: stack=0x557e196bcde0 is done 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d8e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 collect(): s=0x557e1957ee80 stack=0x557e196bcde0 is complete 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 15 stack 0x557e1957ee80 end 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 stack->operate() returned ret=-22 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is done 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196c4c90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 4 starting trim on bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196b95b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196d8e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196b95b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 25577104474bb4beb7dd09b782fa711f2570787cc7a621082aa906de3639e027 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:25577104474bb4beb7dd09b782fa711f2570787cc7a621082aa906de3639e027 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = fe6e5e670853feb4b6847378ef34444333e83bbd7379bb8ded6db2af00aa288b 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=fe6e5e670853feb4b6847378ef34444333e83bbd7379bb8ded6db2af00aa288b 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=38, curl_handle=0x557e196861c0 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8eb2ffd640 20 link_request req_data=0x7f8ee000f390 req_data->id=38, curl_handle=0x557e196861c0 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 run: stack=0x557e196bcde0 is io blocked 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196d8e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 enqueued request req=0x557e196c3c00 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 req: 0x557e196c3c00 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is io blocked 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f0effd640 20 dequeued request req=0x557e196c3c00 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f0effd640 20 RGWWQ: empty 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f0effd640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 : miss 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f0effd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f0effd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f0effd640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 info.flags=0x17 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f0effd640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 to cache LRU end 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f0effd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f0effd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f0effd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f0effd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-52 : miss 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.721+0000 7f8f0effd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f0effd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f0effd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-52 info.flags=0x0 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f0effd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-52 to cache LRU end 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f0effd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-52 : miss 2026-03-20T11:45:37.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f0effd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f0effd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f0effd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-52 info.flags=0x0 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f0effd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-52 to cache LRU end 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f0effd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12): adding source pipe: {s={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f0effd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12): adding dest pipe: {s={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f0effd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f0effd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f0effd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196d8e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196d8e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196d8e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196d8e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e19700e90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196fe0d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196fe0d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 9e56fbdc74b635fda523b38415472de20dd13be6a9bdb7aaeb9910e114ee40e8 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:9e56fbdc74b635fda523b38415472de20dd13be6a9bdb7aaeb9910e114ee40e8 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = df4a7b172f15820ddff6bcd26a24564445c83f3ec7387ebc8b151d9bb118e0a0 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=df4a7b172f15820ddff6bcd26a24564445c83f3ec7387ebc8b151d9bb118e0a0 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=39, curl_handle=0x557e196b2160 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8eb2ffd640 20 link_request req_data=0x557e196b2870 req_data->id=39, curl_handle=0x557e196b2160 2026-03-20T11:45:37.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.722+0000 7f8f59593900 20 run: stack=0x557e196d8570 is io blocked 2026-03-20T11:45:37.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e196c44d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e196c44d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e196c44d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e196c44d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1920feb0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 15 stack 0x557e19701570 end 2026-03-20T11:45:37.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 20 run: stack=0x557e19701570 is done 2026-03-20T11:45:37.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19210960:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 20 collect(): s=0x557e196ad300 stack=0x557e19701570 is complete 2026-03-20T11:45:37.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) markers=[,00000000001.77.6,,,,00000000001.65.6,,,00000000001.20.6,,00000000001.28.6], shards=11 2026-03-20T11:45:37.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196d2940:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 10 trimming bilog shard 1 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.77.6 2026-03-20T11:45:37.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 10 trimming bilog shard 5 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.65.6 2026-03-20T11:45:37.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 10 trimming bilog shard 8 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.20.6 2026-03-20T11:45:37.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 10 trimming bilog shard 10 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.28.6 2026-03-20T11:45:37.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e192c61f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d39d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e1920fd10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196d05e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e192c61f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.1 2026-03-20T11:45:37.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 20 run: stack=0x557e19701570 is io blocked 2026-03-20T11:45:37.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d39d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.5 2026-03-20T11:45:37.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 20 run: stack=0x557e196be570 is io blocked 2026-03-20T11:45:37.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e1920fd10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.8 2026-03-20T11:45:37.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 20 run: stack=0x557e196d3220 is io blocked 2026-03-20T11:45:37.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196d05e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.10 2026-03-20T11:45:37.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.757+0000 7f8f59593900 20 run: stack=0x557e196baa80 is io blocked 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196d05e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196d05e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e1920fd10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d39d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e192c61f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196d05e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e1920fd10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d39d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e192c61f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196d05e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 15 stack 0x557e196baa80 end 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 run: stack=0x557e196baa80 is done 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e1920fd10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d39d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e192c61f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196d2940:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e1920fd10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 15 stack 0x557e196d3220 end 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 run: stack=0x557e196d3220 is done 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d39d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 15 stack 0x557e196be570 end 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 run: stack=0x557e196be570 is done 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e192c61f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 15 stack 0x557e19701570 end 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 run: stack=0x557e19701570 is done 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196d2940:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196d2940:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196d2940:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19210960:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 15 stack 0x557e196ad300 end 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 run: stack=0x557e196ad300 is done 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19210960:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 4 starting trim on bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 enqueued request req=0x557e196d2f30 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 req: 0x557e196d2f30 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f59593900 20 run: stack=0x557e196ad300 is io blocked 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f0e7fc640 20 dequeued request req=0x557e196d2f30 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f0e7fc640 20 RGWWQ: empty 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f0e7fc640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 : miss 2026-03-20T11:45:37.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f0e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f0e7fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:37.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f0e7fc640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 info.flags=0x17 2026-03-20T11:45:37.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f0e7fc640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 to cache LRU end 2026-03-20T11:45:37.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f0e7fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f0e7fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f0e7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:45:37.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f0e7fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-42 : miss 2026-03-20T11:45:37.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.760+0000 7f8f0e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.761+0000 7f8f0e7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.761+0000 7f8f0e7fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-42 info.flags=0x0 2026-03-20T11:45:37.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.761+0000 7f8f0e7fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-42 to cache LRU end 2026-03-20T11:45:37.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.761+0000 7f8f0e7fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-42 : miss 2026-03-20T11:45:37.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.761+0000 7f8f0e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.761+0000 7f8f0e7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.761+0000 7f8f0e7fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-42 info.flags=0x0 2026-03-20T11:45:37.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.761+0000 7f8f0e7fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-42 to cache LRU end 2026-03-20T11:45:37.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.761+0000 7f8f0e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2): adding source pipe: {s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.761+0000 7f8f0e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2): adding dest pipe: {s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.761+0000 7f8f0e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.761+0000 7f8f0e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.761+0000 7f8f0e7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:45:37.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.761+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.761+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.761+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.761+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.761+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19210960:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.762+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e19701760:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.762+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19210960:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.762+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.762+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e1920a900:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.762+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e1920a900:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.762+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.762+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.762+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.762+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.762+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.762+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.762+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 684b659920e120cf08751e51ae796eb70992c0ee2f161346a2f247aff9972c54 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.762+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr:684b659920e120cf08751e51ae796eb70992c0ee2f161346a2f247aff9972c54 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.762+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.762+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.762+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.762+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.762+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 77332d6dff217c0fc2030115aefdcbc4d82b035503b93d06a1a2ab3346bf5a80 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.762+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=77332d6dff217c0fc2030115aefdcbc4d82b035503b93d06a1a2ab3346bf5a80 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.762+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.762+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.762+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.762+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=40, curl_handle=0x557e19681800 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.762+0000 7f8eb2ffd640 20 link_request req_data=0x557e196823d0 req_data->id=40, curl_handle=0x557e19681800 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.762+0000 7f8f59593900 20 run: stack=0x557e196d3220 is io blocked 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196cffb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196cffb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196da040:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f59593900 15 stack 0x557e196cf7c0 end 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f59593900 20 run: stack=0x557e196cf7c0 is done 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f59593900 20 collect(): s=0x557e19688770 stack=0x557e196cf7c0 is complete 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f59593900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f59593900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1967c360:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f59593900 15 stack 0x557e19688770 end 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f59593900 20 stack->operate() returned ret=-22 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f59593900 20 run: stack=0x557e19688770 is done 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f59593900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f59593900 4 starting trim on bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f59593900 20 enqueued request req=0x557e19671360 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f59593900 20 req: 0x557e19671360 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f59593900 20 run: stack=0x557e19688770 is io blocked 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f0dffb640 20 dequeued request req=0x557e19671360 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f0dffb640 20 RGWWQ: empty 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f0dffb640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 : miss 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f0dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f0dffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f0dffb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 info.flags=0x17 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f0dffb640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 to cache LRU end 2026-03-20T11:45:37.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f0dffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f0dffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f0dffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:45:37.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f0dffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-2 : miss 2026-03-20T11:45:37.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f0dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.762 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:37.763+0000 7fd06db29640 -1 req 7700865967869789397 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:37.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f0dffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f0dffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-2 info.flags=0x0 2026-03-20T11:45:37.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f0dffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-2 to cache LRU end 2026-03-20T11:45:37.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f0dffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-2 : miss 2026-03-20T11:45:37.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.763+0000 7f8f0dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f0dffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f0dffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-2 info.flags=0x0 2026-03-20T11:45:37.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f0dffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-2 to cache LRU end 2026-03-20T11:45:37.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f0dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2): adding source pipe: {s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f0dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2): adding dest pipe: {s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f0dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f0dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f0dffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:45:37.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196c44d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 20 run: stack=0x557e19688770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196da040:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196da040:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 1f2b1bd98a2667f95e5c52ef342cbe3f4ffef4d65351f383feaf5cd054157d8b 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:1f2b1bd98a2667f95e5c52ef342cbe3f4ffef4d65351f383feaf5cd054157d8b 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 377733dbf4fd13dca7abf94b52a7389b7f042bd3e5cf8757c315927de2bb809d 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=377733dbf4fd13dca7abf94b52a7389b7f042bd3e5cf8757c315927de2bb809d 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=41, curl_handle=0x557e196b6260 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8eb2ffd640 20 link_request req_data=0x557e196b6970 req_data->id=41, curl_handle=0x557e196b6260 2026-03-20T11:45:37.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.764+0000 7f8f59593900 20 run: stack=0x557e196cf7c0 is io blocked 2026-03-20T11:45:37.763 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:37.764+0000 7fcfd09ef640 -1 req 15246391448924591894 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196b95b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196b95b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196fe0d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196b95b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196fe0d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196b95b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196fe0d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196c4c90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 15 stack 0x557e196bcde0 end 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 run: stack=0x557e196bcde0 is done 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196fe0d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 collect(): s=0x557e196c7af0 stack=0x557e196bcde0 is complete 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-20[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25]) markers=[,,,,,,,,00000000005.42.14,,], shards=11 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e19700e90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 15 stack 0x557e196d8570 end 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 run: stack=0x557e196d8570 is done 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196c4c90:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 10 trimming bilog shard 8 of :dipepu-20[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25]) at marker 00000000005.42.14 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 collect(): s=0x557e1957ee80 stack=0x557e196d8570 is complete 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) markers=[00000000004.19.6,00000000002.34.1,00000000004.47.6,00000000002.22.1,00000000004.31.6,00000000002.67.1,00000000002.32.1,00000000004.16.6,00000000002.26.1,00000000002.76.1,00000000002.54.1], shards=11 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e19700b80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196bb270:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 10 trimming bilog shard 0 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000004.19.6 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 10 trimming bilog shard 1 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.34.1 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 10 trimming bilog shard 2 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000004.47.6 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 10 trimming bilog shard 3 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.22.1 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 10 trimming bilog shard 4 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000004.31.6 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 10 trimming bilog shard 5 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.67.1 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 10 trimming bilog shard 6 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.32.1 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 10 trimming bilog shard 7 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000004.16.6 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 10 trimming bilog shard 8 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.26.1 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 10 trimming bilog shard 9 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.76.1 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 10 trimming bilog shard 10 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.54.1 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e19700b80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25.8 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 run: stack=0x557e196bcde0 is io blocked 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196d4200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e192c6ab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e196d6b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e192121d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1966eb30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1966f330:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e1920c450:op=0x557e196cffb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e196d4a00:op=0x557e196d07b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e19700900:op=0x557e196d0fb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e196fdd70:op=0x557e19702cf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e197034f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196d4200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.0 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 run: stack=0x557e196d8570 is io blocked 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e192c6ab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.1 2026-03-20T11:45:37.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 run: stack=0x557e196be570 is io blocked 2026-03-20T11:45:37.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e196d6b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.765+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.2 2026-03-20T11:45:37.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.766+0000 7f8f59593900 20 run: stack=0x557e19701570 is io blocked 2026-03-20T11:45:37.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.766+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e192121d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.766+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.3 2026-03-20T11:45:37.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.766+0000 7f8f59593900 20 run: stack=0x557e196baa80 is io blocked 2026-03-20T11:45:37.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.766+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1966eb30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.766+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.4 2026-03-20T11:45:37.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.766+0000 7f8f59593900 20 run: stack=0x557e196c3030 is io blocked 2026-03-20T11:45:37.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.766+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1966f330:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.766+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.5 2026-03-20T11:45:37.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.767+0000 7f8f59593900 20 run: stack=0x557e196700a0 is io blocked 2026-03-20T11:45:37.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.767+0000 7f8f59593900 20 cr:s=0x557e1920c450:op=0x557e196cffb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.767+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.6 2026-03-20T11:45:37.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.767+0000 7f8f59593900 20 run: stack=0x557e1920c450 is io blocked 2026-03-20T11:45:37.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.767+0000 7f8f59593900 20 cr:s=0x557e196d4a00:op=0x557e196d07b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.767+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.7 2026-03-20T11:45:37.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.768+0000 7f8f59593900 20 run: stack=0x557e196d4a00 is io blocked 2026-03-20T11:45:37.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.768+0000 7f8f59593900 20 cr:s=0x557e19700900:op=0x557e196d0fb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.768+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.8 2026-03-20T11:45:37.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.768+0000 7f8f59593900 20 run: stack=0x557e19700900 is io blocked 2026-03-20T11:45:37.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.768+0000 7f8f59593900 20 cr:s=0x557e196fdd70:op=0x557e19702cf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.768+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.9 2026-03-20T11:45:37.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 run: stack=0x557e196fdd70 is io blocked 2026-03-20T11:45:37.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e197034f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.10 2026-03-20T11:45:37.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 run: stack=0x557e196c4280 is io blocked 2026-03-20T11:45:37.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196d4200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e192c6ab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e19700b80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e192121d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e196d6b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1966eb30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1966f330:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196d4a00:op=0x557e196d07b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e1920c450:op=0x557e196cffb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196d4200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e192c6ab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e19700b80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e192121d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e196d6b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1966eb30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1966f330:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196d4a00:op=0x557e196d07b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e1920c450:op=0x557e196cffb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196d4200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e192c6ab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e19700b80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e192121d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e196d6b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1966eb30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1966f330:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196d4a00:op=0x557e196d07b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e1920c450:op=0x557e196cffb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196d4200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 15 stack 0x557e196d8570 end 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 run: stack=0x557e196d8570 is done 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e192c6ab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 15 stack 0x557e196be570 end 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 run: stack=0x557e196be570 is done 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e19700b80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 15 stack 0x557e196bcde0 end 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 run: stack=0x557e196bcde0 is done 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e192121d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 15 stack 0x557e196baa80 end 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 run: stack=0x557e196baa80 is done 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e196d6b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 15 stack 0x557e19701570 end 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 run: stack=0x557e19701570 is done 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1966eb30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 15 stack 0x557e196c3030 end 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 run: stack=0x557e196c3030 is done 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1966f330:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 15 stack 0x557e196700a0 end 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 run: stack=0x557e196700a0 is done 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196d4a00:op=0x557e196d07b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 15 stack 0x557e196d4a00 end 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 run: stack=0x557e196d4a00 is done 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e1920c450:op=0x557e196cffb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 15 stack 0x557e1920c450 end 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 run: stack=0x557e1920c450 is done 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196bb270:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196c4c90:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196bb270:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 15 stack 0x557e196c7af0 end 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is done 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196bb270:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196bb270:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 4 starting trim on bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196bb270:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196bb270:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 enqueued request req=0x557e196ad110 2026-03-20T11:45:37.768 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.769 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 req: 0x557e196ad110 2026-03-20T11:45:37.769 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is io blocked 2026-03-20T11:45:37.769 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196bb270:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.769 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196bb270:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.769 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.769 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f0d7fa640 20 dequeued request req=0x557e196ad110 2026-03-20T11:45:37.769 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f0d7fa640 20 RGWWQ: empty 2026-03-20T11:45:37.769 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f0d7fa640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 : miss 2026-03-20T11:45:37.769 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.769+0000 7f8f0d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f0d7fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f0d7fa640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 info.flags=0x17 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f0d7fa640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 to cache LRU end 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f0d7fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f0d7fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f0d7fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f0d7fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-57 : miss 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f0d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 cr:s=0x557e19700900:op=0x557e196d0fb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 cr:s=0x557e19700900:op=0x557e196d0fb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 cr:s=0x557e19700900:op=0x557e196d0fb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 cr:s=0x557e19700900:op=0x557e196d0fb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 15 stack 0x557e19700900 end 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 run: stack=0x557e19700900 is done 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196bb270:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f0d7fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f0d7fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-57 info.flags=0x0 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f0d7fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-57 to cache LRU end 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f0d7fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-57 : miss 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f0d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 cr:s=0x557e196fdd70:op=0x557e19702cf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 cr:s=0x557e196fdd70:op=0x557e19702cf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 cr:s=0x557e196fdd70:op=0x557e19702cf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 cr:s=0x557e196fdd70:op=0x557e19702cf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 15 stack 0x557e196fdd70 end 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 run: stack=0x557e196fdd70 is done 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196bb270:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f0d7fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f0d7fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-57 info.flags=0x0 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f0d7fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-57 to cache LRU end 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f0d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17): adding source pipe: {s={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f0d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17): adding dest pipe: {s={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f0d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f0d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f0d7fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.770 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c4c90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1920bd20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1920bd20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 899a798920f6d7a053506cc3d363920909c27766ddd4264d75e61d8faeae6b9d 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:899a798920f6d7a053506cc3d363920909c27766ddd4264d75e61d8faeae6b9d 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 655b7563326042c54c497149cc785a33119105e5d2d3f7ab7c2bf71f5d5ea725 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=655b7563326042c54c497149cc785a33119105e5d2d3f7ab7c2bf71f5d5ea725 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=42, curl_handle=0x557e196b2160 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.770+0000 7f8eb2ffd640 20 link_request req_data=0x557e196b2870 req_data->id=42, curl_handle=0x557e196b2160 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f59593900 20 run: stack=0x557e196700a0 is io blocked 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e197034f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e197034f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e197034f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e197034f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f59593900 15 stack 0x557e196c4280 end 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f59593900 20 run: stack=0x557e196c4280 is done 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e196bb270:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f59593900 20 cr:s=0x557e1957ee80:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f59593900 15 stack 0x557e1957ee80 end 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f59593900 20 run: stack=0x557e1957ee80 is done 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f59593900 4 starting trim on bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f59593900 20 enqueued request req=0x557e19702960 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f59593900 20 req: 0x557e19702960 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f59593900 20 run: stack=0x557e196c4280 is io blocked 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f0cff9640 20 dequeued request req=0x557e19702960 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f0cff9640 20 RGWWQ: empty 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f0cff9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 : miss 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f0cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f0cff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f0cff9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 info.flags=0x17 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f0cff9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 to cache LRU end 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f0cff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f0cff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f0cff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f0cff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-29 : miss 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f0cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f0cff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f0cff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-29 info.flags=0x0 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f0cff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-29 to cache LRU end 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f0cff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-29 : miss 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f0cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f0cff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f0cff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-29 info.flags=0x0 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f0cff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-29 to cache LRU end 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f0cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34): adding source pipe: {s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f0cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34): adding dest pipe: {s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f0cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.771+0000 7f8f0cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f0cff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196b95b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 20 run: stack=0x557e196c4280 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1920e460:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1920e460:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.772 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.773 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.773 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.773 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = d46eee25a50c1a6fbd97ead78af0f2053dc99e78c4e06c6cb139118ec39371f5 2026-03-20T11:45:37.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.773 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.773 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.773 INFO:teuthology.orchestra.run.vm02.stderr:d46eee25a50c1a6fbd97ead78af0f2053dc99e78c4e06c6cb139118ec39371f5 2026-03-20T11:45:37.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 02e3d4fa5d19d7c4560a469f804eb783a70e612bb050e61bb3c8222cfedd1e46 2026-03-20T11:45:37.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=02e3d4fa5d19d7c4560a469f804eb783a70e612bb050e61bb3c8222cfedd1e46 2026-03-20T11:45:37.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=43, curl_handle=0x557e196861c0 2026-03-20T11:45:37.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8eb2ffd640 20 link_request req_data=0x7f8ee000f390 req_data->id=43, curl_handle=0x557e196861c0 2026-03-20T11:45:37.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.772+0000 7f8f59593900 20 run: stack=0x557e196c3030 is io blocked 2026-03-20T11:45:37.773 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:37.772+0000 7fcfd9200640 -1 req 2679606240640571927 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e1920a900:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e1920a900:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e19701760:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f59593900 15 stack 0x557e196d3220 end 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f59593900 20 run: stack=0x557e196d3220 is done 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19210960:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f59593900 20 collect(): s=0x557e196ad300 stack=0x557e196d3220 is complete 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f59593900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f59593900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19210960:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f59593900 15 stack 0x557e196ad300 end 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f59593900 20 stack->operate() returned ret=-22 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f59593900 20 run: stack=0x557e196ad300 is done 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f59593900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19210960:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f59593900 4 starting trim on bucket=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f59593900 20 enqueued request req=0x557e196d2f30 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f59593900 20 req: 0x557e196d2f30 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f59593900 20 run: stack=0x557e196ad300 is io blocked 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f0c7f8640 20 dequeued request req=0x557e196d2f30 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f0c7f8640 20 RGWWQ: empty 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f0c7f8640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 : miss 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f0c7f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f0c7f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=408 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f0c7f8640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 info.flags=0x17 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f0c7f8640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 to cache LRU end 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f0c7f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f0c7f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=171 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f0c7f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f0c7f8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.testx/tenanted-bucket : miss 2026-03-20T11:45:37.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.805+0000 7f8f0c7f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196da040:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196da040:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196c44d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 15 stack 0x557e196cf7c0 end 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 run: stack=0x557e196cf7c0 is done 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 collect(): s=0x557e19688770 stack=0x557e196cf7c0 is complete 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f0c7f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f0c7f8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.testx/tenanted-bucket info.flags=0x0 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f0c7f8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.testx/tenanted-bucket to cache LRU end 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1967c360:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 15 stack 0x557e19688770 end 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f0c7f8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.testx/tenanted-bucket : miss 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f0c7f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 stack->operate() returned ret=-22 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 run: stack=0x557e19688770 is done 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 4 starting trim on bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e19700130:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e19700130:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 enqueued request req=0x557e19671360 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 req: 0x557e19671360 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 run: stack=0x557e196cf7c0 is io blocked 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f0bff7640 20 dequeued request req=0x557e19671360 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f0bff7640 20 RGWWQ: empty 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f0bff7640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 : miss 2026-03-20T11:45:37.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f0bff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f0c7f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f0c7f8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.testx/tenanted-bucket info.flags=0x0 2026-03-20T11:45:37.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f0c7f8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.testx/tenanted-bucket to cache LRU end 2026-03-20T11:45:37.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f0c7f8640 20 rgw async rados processor: reflect(): flow manager (bucket=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3): adding source pipe: {s={b=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f0c7f8640 20 rgw async rados processor: reflect(): flow manager (bucket=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3): adding dest pipe: {s={b=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f0c7f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f0c7f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f0c7f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 2026-03-20T11:45:37.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f0bff7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196c2540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f0bff7640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 info.flags=0x17 2026-03-20T11:45:37.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f0bff7640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 to cache LRU end 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f0bff7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f0bff7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19210960:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f0bff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f0bff7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-23 : miss 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f0bff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e196c00a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19210960:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e196c44d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e196c44d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = ee47b7a8d0aac9b9aedb73616576ebc7861affb16e37b806183d22658b45b0f0 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:ee47b7a8d0aac9b9aedb73616576ebc7861affb16e37b806183d22658b45b0f0 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = f1c229fde9df38db52481c80bd2d4b21f91995036e47be98c219548b30e4ad75 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=f1c229fde9df38db52481c80bd2d4b21f91995036e47be98c219548b30e4ad75 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=44, curl_handle=0x557e196b6260 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.806+0000 7f8eb2ffd640 20 link_request req_data=0x557e196b6970 req_data->id=44, curl_handle=0x557e196b6260 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 20 run: stack=0x557e196d3220 is io blocked 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f0bff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f0bff7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-23 info.flags=0x0 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f0bff7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-23 to cache LRU end 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f0bff7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-23 : miss 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f0bff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f0bff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f0bff7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-23 info.flags=0x0 2026-03-20T11:45:37.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f0bff7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-23 to cache LRU end 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f0bff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28): adding source pipe: {s={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f0bff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28): adding dest pipe: {s={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f0bff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f0bff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f0bff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e19700130:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e19700130:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e19700130:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e19700130:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196da040:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 20 run: stack=0x557e196cf7c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196fe0d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196fe0d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 2a11693b4724586ca30cdcf12d9aafd6151b688a79664e9f4185b7fa469131c9 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2a11693b4724586ca30cdcf12d9aafd6151b688a79664e9f4185b7fa469131c9 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 86898acf04c22455a5d46a61151dd90cf4187a67baa71756c4274b91a0cd9ef8 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=86898acf04c22455a5d46a61151dd90cf4187a67baa71756c4274b91a0cd9ef8 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=45, curl_handle=0x557e19681800 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8eb2ffd640 20 link_request req_data=0x557e196823d0 req_data->id=45, curl_handle=0x557e19681800 2026-03-20T11:45:37.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.807+0000 7f8f59593900 20 run: stack=0x557e196baa80 is io blocked 2026-03-20T11:45:37.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.813+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1920bd20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.813+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1920bd20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.813+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1920bd20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.813+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1920bd20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.813+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c4c90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.813+0000 7f8f59593900 15 stack 0x557e196700a0 end 2026-03-20T11:45:37.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.813+0000 7f8f59593900 20 run: stack=0x557e196700a0 is done 2026-03-20T11:45:37.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.813+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.813+0000 7f8f59593900 20 collect(): s=0x557e196c7af0 stack=0x557e196700a0 is complete 2026-03-20T11:45:37.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.813+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.813+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-57[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17]) markers=[,00000000001.41.6,,,,00000000001.23.6,,,,,], shards=11 2026-03-20T11:45:37.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.813+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d1210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.813+0000 7f8f59593900 10 trimming bilog shard 1 of :dipepu-57[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17]) at marker 00000000001.41.6 2026-03-20T11:45:37.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.813+0000 7f8f59593900 10 trimming bilog shard 5 of :dipepu-57[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17]) at marker 00000000001.23.6 2026-03-20T11:45:37.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.813+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.813+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196d4200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.813+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1966f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.813+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196d4200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.813+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17.1 2026-03-20T11:45:37.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.813+0000 7f8f59593900 20 run: stack=0x557e196700a0 is io blocked 2026-03-20T11:45:37.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.813+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1966f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.813+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17.5 2026-03-20T11:45:37.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.813+0000 7f8f59593900 20 run: stack=0x557e19701570 is io blocked 2026-03-20T11:45:37.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1920e460:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1920e460:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196b95b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 15 stack 0x557e196c3030 end 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 run: stack=0x557e196c3030 is done 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 collect(): s=0x557e196c4280 stack=0x557e196c3030 is complete 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 15 stack 0x557e196c4280 end 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 stack->operate() returned ret=-22 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 run: stack=0x557e196c4280 is done 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 4 starting trim on bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e192c61f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e192c61f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 enqueued request req=0x557e19702960 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 req: 0x557e19702960 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 run: stack=0x557e196c4280 is io blocked 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f0b7f6640 20 dequeued request req=0x557e19702960 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f0b7f6640 20 RGWWQ: empty 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f0b7f6640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 : miss 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f0b7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196d4200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196d4200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196d4200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196d4200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 15 stack 0x557e196700a0 end 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 run: stack=0x557e196700a0 is done 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d1210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f0b7f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f0b7f6640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 info.flags=0x17 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f0b7f6640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 to cache LRU end 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f0b7f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f0b7f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f0b7f6640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f0b7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f0b7f6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-34 : miss 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f0b7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1966f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1966f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1966f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1966f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 15 stack 0x557e19701570 end 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 run: stack=0x557e19701570 is done 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d1210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 15 stack 0x557e196c7af0 end 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is done 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 4 starting trim on bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d4200:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d4200:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 enqueued request req=0x557e196ff0f0 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 req: 0x557e196ff0f0 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is io blocked 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f0aff5640 20 dequeued request req=0x557e196ff0f0 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f0aff5640 20 RGWWQ: empty 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f0aff5640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 : miss 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f0aff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f0b7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f0b7f6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-34 info.flags=0x0 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f0b7f6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-34 to cache LRU end 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f0b7f6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-34 : miss 2026-03-20T11:45:37.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.814+0000 7f8f0b7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0aff5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0aff5640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 info.flags=0x17 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0aff5640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 to cache LRU end 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0aff5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0aff5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0aff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0aff5640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-54 : miss 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0aff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0b7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0b7f6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-34 info.flags=0x0 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0b7f6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-34 to cache LRU end 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0b7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39): adding source pipe: {s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0b7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39): adding dest pipe: {s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0b7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0b7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0b7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e192c61f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e192c61f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e192c61f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e192c61f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e196b95b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 run: stack=0x557e196c4280 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1920bd20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1920bd20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0aff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0aff5640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-54 info.flags=0x0 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0aff5640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-54 to cache LRU end 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0aff5640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-54 : miss 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0aff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = ce8e157a191750caf30251b15aaad7207705fa0f9a2da09352886639e0b28dc1 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:ce8e157a191750caf30251b15aaad7207705fa0f9a2da09352886639e0b28dc1 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 5a536b7ff00691bc3740523ebe640b94a6faeed5100299ce29e589a39515b19c 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5a536b7ff00691bc3740523ebe640b94a6faeed5100299ce29e589a39515b19c 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=46, curl_handle=0x557e196861c0 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8eb2ffd640 20 link_request req_data=0x7f8ee000f390 req_data->id=46, curl_handle=0x557e196861c0 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 run: stack=0x557e19701570 is io blocked 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0aff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0aff5640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-54 info.flags=0x0 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0aff5640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-54 to cache LRU end 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0aff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14): adding source pipe: {s={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0aff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14): adding dest pipe: {s={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0aff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0aff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f0aff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d4200:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d4200:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d4200:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d4200:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c4c90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e19700a10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e19700a10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.815+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.816+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.816+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.816+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.816+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.816+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.815 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.816 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.816 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.816 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.816 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.816 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.816 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.816 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.816+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = f9529459bd022b99e2a3393a1a41999178b03467c0709274628d6ffc78a8df78 2026-03-20T11:45:37.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.816+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.816 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.816 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.816 INFO:teuthology.orchestra.run.vm02.stderr:f9529459bd022b99e2a3393a1a41999178b03467c0709274628d6ffc78a8df78 2026-03-20T11:45:37.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.816+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.816+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.816+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.816+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.816+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = cea36657ff28056cf1a2ccb40c417adc9d5f252ede3a5589a0bd33b313017046 2026-03-20T11:45:37.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.816+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=cea36657ff28056cf1a2ccb40c417adc9d5f252ede3a5589a0bd33b313017046 2026-03-20T11:45:37.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.816+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.816+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.816+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.816+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=47, curl_handle=0x557e196b2160 2026-03-20T11:45:37.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.816+0000 7f8eb2ffd640 20 link_request req_data=0x557e196b2870 req_data->id=47, curl_handle=0x557e196b2160 2026-03-20T11:45:37.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.816+0000 7f8f59593900 20 run: stack=0x557e196700a0 is io blocked 2026-03-20T11:45:37.816 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:37.816+0000 7fd012a73640 -1 req 4863787629316947868 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e196c44d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e196c44d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e196c44d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e196c44d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e196c00a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 15 stack 0x557e196d3220 end 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 run: stack=0x557e196d3220 is done 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19210960:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 collect(): s=0x557e196ad300 stack=0x557e196d3220 is complete 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 10 min_generation is 1 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196d1210:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.0 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.1 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.2 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.3 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.4 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.5 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.6 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.7 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.8 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.9 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.10 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e196c00a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1920c920:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920ab70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196c44d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e1920e460:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e1966f1f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e1920ea50:op=0x557e196d5aa0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e196d6930:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e1920b2d0:op=0x557e196ffd20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e1920ff40:op=0x557e19701760:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e192c4bb0:op=0x557e19702b50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e196c00a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 run: stack=0x557e196d3220 is io blocked 2026-03-20T11:45:37.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1920c920:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 run: stack=0x557e196c3030 is io blocked 2026-03-20T11:45:37.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920ab70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 run: stack=0x557e196be570 is io blocked 2026-03-20T11:45:37.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196c44d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 run: stack=0x557e196bcde0 is io blocked 2026-03-20T11:45:37.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e1920e460:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 run: stack=0x557e196d8570 is io blocked 2026-03-20T11:45:37.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e1966f1f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 run: stack=0x557e196c5e20 is io blocked 2026-03-20T11:45:37.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e1920ea50:op=0x557e196d5aa0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 run: stack=0x557e1920ea50 is io blocked 2026-03-20T11:45:37.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e196d6930:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 run: stack=0x557e19686770 is io blocked 2026-03-20T11:45:37.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e1920b2d0:op=0x557e196ffd20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 run: stack=0x557e1920b2d0 is io blocked 2026-03-20T11:45:37.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e1920ff40:op=0x557e19701760:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 run: stack=0x557e1920ff40 is io blocked 2026-03-20T11:45:37.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 cr:s=0x557e192c4bb0:op=0x557e19702b50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.850+0000 7f8f59593900 20 run: stack=0x557e192c4bb0 is io blocked 2026-03-20T11:45:37.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.852+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196fe0d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196fe0d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920ab70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196c44d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e196d6930:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e196c00a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e1966f1f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196fe0d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920ab70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196c44d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e196d6930:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e196c00a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e1966f1f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196fe0d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920ab70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196c44d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e196d6930:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e196c00a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e1966f1f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196da040:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 15 stack 0x557e196baa80 end 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 run: stack=0x557e196baa80 is done 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920ab70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 15 stack 0x557e196be570 end 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 run: stack=0x557e196be570 is done 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e196c44d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 15 stack 0x557e196bcde0 end 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 run: stack=0x557e196bcde0 is done 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e196d6930:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 15 stack 0x557e19686770 end 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 run: stack=0x557e19686770 is done 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e196c00a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 15 stack 0x557e196d3220 end 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 run: stack=0x557e196d3220 is done 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e1966f1f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 15 stack 0x557e196c5e20 end 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 run: stack=0x557e196c5e20 is done 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 collect(): s=0x557e196cf7c0 stack=0x557e196baa80 is complete 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-23[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28]) markers=[,,,,,,,,00000000003.16.7,,], shards=11 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196d1210:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1920c920:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196c44d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 10 trimming bilog shard 8 of :dipepu-23[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28]) at marker 00000000003.16.7 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 run: stack=0x557e196cf7c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196d1210:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1920c920:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e1970dc80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196d1210:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1920c920:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e192c4bb0:op=0x557e19702b50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e1920e460:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e1920ea50:op=0x557e196d5aa0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e1920ff40:op=0x557e19701760:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e1970dc80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28.8 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 run: stack=0x557e196d3220 is io blocked 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196d1210:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1920c920:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 15 stack 0x557e196c3030 end 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 run: stack=0x557e196c3030 is done 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e192c4bb0:op=0x557e19702b50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e1920e460:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e1920ea50:op=0x557e196d5aa0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e1920ff40:op=0x557e19701760:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e1920b2d0:op=0x557e196ffd20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196d1210:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e192c4bb0:op=0x557e19702b50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e1920e460:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e1920ea50:op=0x557e196d5aa0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e1920ff40:op=0x557e19701760:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e1920b2d0:op=0x557e196ffd20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196d1210:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e192c4bb0:op=0x557e19702b50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 15 stack 0x557e192c4bb0 end 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 run: stack=0x557e192c4bb0 is done 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e1920e460:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 15 stack 0x557e196d8570 end 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 run: stack=0x557e196d8570 is done 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e1920ea50:op=0x557e196d5aa0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 15 stack 0x557e1920ea50 end 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 run: stack=0x557e1920ea50 is done 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e1920ff40:op=0x557e19701760:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 15 stack 0x557e1920ff40 end 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 run: stack=0x557e1920ff40 is done 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e1920b2d0:op=0x557e196ffd20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196d1210:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e1920b2d0:op=0x557e196ffd20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 15 stack 0x557e1920b2d0 end 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 run: stack=0x557e1920b2d0 is done 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196d1210:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196d1210:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196d1210:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196d1210:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19210960:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196d1210:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196d1210:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 enqueued request req=0x557e192c7740 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 req: 0x557e192c7740 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f59593900 20 run: stack=0x557e196ad300 is io blocked 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f0a7f4640 20 dequeued request req=0x557e192c7740 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.853+0000 7f8f0a7f4640 20 RGWWQ: empty 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e1970dc80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e1970dc80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e1970dc80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f59593900 20 cr:s=0x557e196d3220:op=0x557e1970dc80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f59593900 15 stack 0x557e196d3220 end 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f59593900 20 run: stack=0x557e196d3220 is done 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e196c44d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f59593900 20 cr:s=0x557e196cf7c0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f59593900 15 stack 0x557e196cf7c0 end 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f59593900 20 run: stack=0x557e196cf7c0 is done 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f59593900 4 starting trim on bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e19706890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e19706890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f59593900 20 enqueued request req=0x557e196b1c70 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f59593900 20 req: 0x557e196b1c70 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f09ff3640 20 dequeued request req=0x557e196b1c70 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f09ff3640 20 RGWWQ: empty 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f09ff3640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 : miss 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f09ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f59593900 20 run: stack=0x557e196d8570 is io blocked 2026-03-20T11:45:37.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f09ff3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f09ff3640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 info.flags=0x17 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f09ff3640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 to cache LRU end 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f09ff3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f09ff3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f09ff3640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f09ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f09ff3640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-32 : miss 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f09ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f0a7f4640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 info.flags=0x11 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f0a7f4640 10 rgw async rados processor: moving a1.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 to cache LRU end 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f0a7f4640 10 rgw async rados processor: distributing notification oid=a1.rgw.control:notify.5 cni=[op: 0, obj: a1.rgw.meta:root:.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3, ofs0, ns] 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f09ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f09ff3640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-32 info.flags=0x0 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f09ff3640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-32 to cache LRU end 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f09ff3640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-32 : miss 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.855+0000 7f8f09ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f09ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f09ff3640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-32 info.flags=0x0 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f09ff3640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-32 to cache LRU end 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f09ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37): adding source pipe: {s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f09ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37): adding dest pipe: {s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f09ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f09ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f09ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f23fff640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 335007449319 cookie 94000080666336 notifier 6226 bl.length()=599 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f23fff640 10 rgw watcher librados: cache put: name=a1.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 info.flags=0x11 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e19706890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e19706890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e19706890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e19706890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f23fff640 10 rgw watcher librados: moving a1.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 to cache LRU end 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e1920c920:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 20 run: stack=0x557e196d8570 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e196c0080:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e196c0080:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.854 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = a2aa57b8c942b19e5cba50722b38f307789b1eaf6d69c5c1e7dc7a6be30ee3d5 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:a2aa57b8c942b19e5cba50722b38f307789b1eaf6d69c5c1e7dc7a6be30ee3d5 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 2d605a436ec307fe4723040ef352645c3660b15e8e8f283fe7c83603403e0163 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2d605a436ec307fe4723040ef352645c3660b15e8e8f283fe7c83603403e0163 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=48, curl_handle=0x557e19681800 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8f59593900 20 run: stack=0x557e19686770 is io blocked 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.856+0000 7f8eb2ffd640 20 link_request req_data=0x557e196823d0 req_data->id=48, curl_handle=0x557e19681800 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.857+0000 7f8f0a7f4640 20 rgw async rados processor: handle_bi_update(): bucket=testx:tenanted-bucket[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3]): orig_sources= new_sources= 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.857+0000 7f8f0a7f4640 20 rgw async rados processor: handle_bi_update(): bucket=testx:tenanted-bucket[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3]): potential sources added=[] removed=[] 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.857+0000 7f8f0a7f4640 20 rgw async rados processor: handle_bi_update(): bucket=testx:tenanted-bucket[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3]): orig_dests= new_dests= 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.857+0000 7f8f0a7f4640 20 rgw async rados processor: handle_bi_update(): bucket=testx:tenanted-bucket[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3]): potential dests added=[] removed=[] 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.857+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196d1210:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.857+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196d1210:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.857+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196d1210:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.857+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196d1210:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.857+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19210960:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.857+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196da040:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.857+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196da040:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.857+0000 7f8f59593900 20 enqueued request req=0x557e191a8f40 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.857+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.857+0000 7f8f59593900 20 req: 0x557e191a8f40 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.857+0000 7f8f097f2640 20 dequeued request req=0x557e191a8f40 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.857+0000 7f8f097f2640 20 RGWWQ: empty 2026-03-20T11:45:37.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.857+0000 7f8f59593900 20 run: stack=0x557e196ad300 is io blocked 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1920bd20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1920bd20:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e196b95b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 15 stack 0x557e19701570 end 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 run: stack=0x557e19701570 is done 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e19700a10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.856 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:37.857+0000 7fd02eaab640 -1 req 14806474965332887607 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 collect(): s=0x557e196c4280 stack=0x557e19701570 is complete 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 15 stack 0x557e196c4280 end 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 stack->operate() returned ret=-22 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 run: stack=0x557e196c4280 is done 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e19700a10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e19700a10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 4 starting trim on bucket=dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e19700a10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19706890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c4c90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 15 stack 0x557e196700a0 end 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 run: stack=0x557e196700a0 is done 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19706890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 enqueued request req=0x557e196ad110 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 req: 0x557e196ad110 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f08ff1640 20 dequeued request req=0x557e196ad110 2026-03-20T11:45:37.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f08ff1640 20 RGWWQ: empty 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f08ff1640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 : miss 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f08ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 run: stack=0x557e196c4280 is io blocked 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 collect(): s=0x557e196c7af0 stack=0x557e196700a0 is complete 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) markers=[00000000004.46.6,00000000002.37.1,00000000004.60.6,00000000002.19.1,00000000004.18.6,00000000002.57.1,00000000002.32.1,00000000004.68.6,00000000002.43.1,00000000002.85.1,00000000002.30.1], shards=11 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d1210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 10 trimming bilog shard 0 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.46.6 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 10 trimming bilog shard 1 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.37.1 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 10 trimming bilog shard 2 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.60.6 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 10 trimming bilog shard 3 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.19.1 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 10 trimming bilog shard 4 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.18.6 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 10 trimming bilog shard 5 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.57.1 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 10 trimming bilog shard 6 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.32.1 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 10 trimming bilog shard 7 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.68.6 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 10 trimming bilog shard 8 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.43.1 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 10 trimming bilog shard 9 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.85.1 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 10 trimming bilog shard 10 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.30.1 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1970d1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1970dc80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1966f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e196d4190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e1920aa80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1970c2a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196ffd00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196d2b20:op=0x557e196d8e00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196d4990:op=0x557e19702b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e1966f9f0:op=0x557e1920bb30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196d8aa0:op=0x557e192121d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1970d1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.0 2026-03-20T11:45:37.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 run: stack=0x557e196700a0 is io blocked 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1970dc80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.1 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 run: stack=0x557e19701570 is io blocked 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1966f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.2 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 run: stack=0x557e196c3030 is io blocked 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e196d4190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.3 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 run: stack=0x557e196c5e20 is io blocked 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e1920aa80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.4 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 run: stack=0x557e196bcde0 is io blocked 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1970c2a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.5 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 run: stack=0x557e196be570 is io blocked 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196ffd00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.6 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 run: stack=0x557e196baa80 is io blocked 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196d2b20:op=0x557e196d8e00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.7 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 run: stack=0x557e196d2b20 is io blocked 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e196d4990:op=0x557e19702b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.8 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 run: stack=0x557e196d4990 is io blocked 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.858+0000 7f8f59593900 20 cr:s=0x557e1966f9f0:op=0x557e1920bb30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.859+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.9 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.859+0000 7f8f097f2640 10 rgw async rados processor: removing a1.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 from cache 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.859+0000 7f8f097f2640 10 rgw async rados processor: distributing notification oid=a1.rgw.control:notify.5 cni=[op: 1, obj: a1.rgw.meta:root:.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3, ofs0, ns] 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.859+0000 7f8f08ff1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=386 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.859+0000 7f8f08ff1640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 info.flags=0x17 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.859+0000 7f8f08ff1640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 to cache LRU end 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.859+0000 7f8f08ff1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.859+0000 7f8f08ff1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.859+0000 7f8f08ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.859+0000 7f8f08ff1640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-5 : miss 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.859+0000 7f8f08ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.859+0000 7f8f08ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.859+0000 7f8f08ff1640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-5 info.flags=0x0 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.859+0000 7f8f08ff1640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-5 to cache LRU end 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.859+0000 7f8f08ff1640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-5 : miss 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.859+0000 7f8f08ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.860+0000 7f8f59593900 20 run: stack=0x557e1966f9f0 is io blocked 2026-03-20T11:45:37.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.860+0000 7f8f59593900 20 cr:s=0x557e196d8aa0:op=0x557e192121d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.860+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.10 2026-03-20T11:45:37.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.860+0000 7f8f59593900 20 run: stack=0x557e196d8aa0 is io blocked 2026-03-20T11:45:37.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.860+0000 7f8f30ff9640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 335007449320 cookie 94000080666336 notifier 6226 bl.length()=217 2026-03-20T11:45:37.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f08ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f08ff1640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-5 info.flags=0x0 2026-03-20T11:45:37.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f08ff1640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-5 to cache LRU end 2026-03-20T11:45:37.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f08ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10): adding source pipe: {s={b=dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f08ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10): adding dest pipe: {s={b=dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f08ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f08ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f08ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 2026-03-20T11:45:37.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1970dc80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1970dc80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1970d1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e1920aa80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19706890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196d4990:op=0x557e19702b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196da040:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1970dc80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1970d1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e1920aa80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19706890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196d4990:op=0x557e19702b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196da040:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1970dc80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 15 stack 0x557e19701570 end 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 run: stack=0x557e19701570 is done 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1970d1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e1920aa80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19706890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196d4990:op=0x557e19702b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196da040:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d1210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1970d1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 15 stack 0x557e196700a0 end 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 run: stack=0x557e196700a0 is done 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e1920aa80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 15 stack 0x557e196bcde0 end 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 run: stack=0x557e196bcde0 is done 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19706890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196d4990:op=0x557e19702b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 15 stack 0x557e196d4990 end 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 run: stack=0x557e196d4990 is done 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196da040:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d1210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19210960:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 trim: trimmed bucket instance testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 15 stack 0x557e196ad300 end 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 run: stack=0x557e196ad300 is done 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d1210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c4c90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 run: stack=0x557e196c4280 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d1210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e19700a10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19210960:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 4 starting trim on bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e19700a10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = cb0d83f7870987ea40c8e41a68d6dfbdf32e1c34f23abb0d67ed1a0b03ba433b 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:cb0d83f7870987ea40c8e41a68d6dfbdf32e1c34f23abb0d67ed1a0b03ba433b 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = f375148ebdba3920b3aeab0831c4d2130ff3147169488c9b770c5df864556753 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=f375148ebdba3920b3aeab0831c4d2130ff3147169488c9b770c5df864556753 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.861+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=49, curl_handle=0x557e196b2160 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 run: stack=0x557e196700a0 is io blocked 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19705be0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1966f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1970c2a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 cr:s=0x557e196d8aa0:op=0x557e192121d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 cr:s=0x557e1966f9f0:op=0x557e1920bb30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e196d4190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19705be0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 enqueued request req=0x557e196ad110 2026-03-20T11:45:37.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8eb2ffd640 20 link_request req_data=0x7f8ebc01cca0 req_data->id=49, curl_handle=0x557e196b2160 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 req: 0x557e196ad110 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f087f0640 20 dequeued request req=0x557e196ad110 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f087f0640 20 RGWWQ: empty 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f087f0640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 : miss 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f087f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 run: stack=0x557e196ad300 is io blocked 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1966f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1970c2a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 cr:s=0x557e196d8aa0:op=0x557e192121d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 cr:s=0x557e1966f9f0:op=0x557e1920bb30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e196d4190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1966f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1970c2a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 cr:s=0x557e196d8aa0:op=0x557e192121d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 cr:s=0x557e1966f9f0:op=0x557e1920bb30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e196d4190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1966f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 15 stack 0x557e196c3030 end 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 run: stack=0x557e196c3030 is done 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1970c2a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 15 stack 0x557e196be570 end 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 run: stack=0x557e196be570 is done 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 cr:s=0x557e196d8aa0:op=0x557e192121d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 15 stack 0x557e196d8aa0 end 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 run: stack=0x557e196d8aa0 is done 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 cr:s=0x557e1966f9f0:op=0x557e1920bb30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 15 stack 0x557e1966f9f0 end 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 run: stack=0x557e1966f9f0 is done 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e196d4190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 15 stack 0x557e196c5e20 end 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 run: stack=0x557e196c5e20 is done 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d1210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d1210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d1210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d1210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d1210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.862+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f087f0640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f087f0640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 info.flags=0x17 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f087f0640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 to cache LRU end 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f087f0640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f087f0640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f087f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f087f0640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-19 : miss 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f087f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f087f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f087f0640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-19 info.flags=0x0 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f087f0640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-19 to cache LRU end 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f087f0640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-19 : miss 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f087f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f087f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f087f0640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-19 info.flags=0x0 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f087f0640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-19 to cache LRU end 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f087f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24): adding source pipe: {s={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f087f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24): adding dest pipe: {s={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f087f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f087f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f087f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 cr:s=0x557e196d2b20:op=0x557e196d8e00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 cr:s=0x557e196d2b20:op=0x557e196d8e00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196ffd00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19705be0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 cr:s=0x557e196d2b20:op=0x557e196d8e00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196ffd00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19705be0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 cr:s=0x557e196d2b20:op=0x557e196d8e00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 15 stack 0x557e196d2b20 end 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 run: stack=0x557e196d2b20 is done 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196ffd00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19705be0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d1210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e196ffd00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 15 stack 0x557e196baa80 end 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 run: stack=0x557e196baa80 is done 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19705be0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196d1210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19210960:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 15 stack 0x557e196c7af0 end 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is done 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196b95b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19210960:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920ff20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 4 starting trim on bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920ff20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.863 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = b16a1638f57c78d59e49981a5933dc26571efe869e7e0ddbf111760b87c6ed80 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:b16a1638f57c78d59e49981a5933dc26571efe869e7e0ddbf111760b87c6ed80 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 66229cf0fad60a8a79483b995275e36990c7e1e98368ab47ad058230cfdd08a2 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=66229cf0fad60a8a79483b995275e36990c7e1e98368ab47ad058230cfdd08a2 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=50, curl_handle=0x557e196861c0 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 run: stack=0x557e196be570 is io blocked 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e19705be0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e19705be0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 enqueued request req=0x557e196c8780 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 req: 0x557e196c8780 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8eb2ffd640 20 link_request req_data=0x557e196b2870 req_data->id=50, curl_handle=0x557e196861c0 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f07fef640 20 dequeued request req=0x557e196c8780 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f07fef640 20 RGWWQ: empty 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f07fef640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 : miss 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f07fef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.863+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is io blocked 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.864+0000 7f8f07fef640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.864+0000 7f8f07fef640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 info.flags=0x17 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.864+0000 7f8f07fef640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 to cache LRU end 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.864+0000 7f8f07fef640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.864+0000 7f8f07fef640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.864+0000 7f8f07fef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.864+0000 7f8f07fef640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-4 : miss 2026-03-20T11:45:37.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.864+0000 7f8f07fef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.864+0000 7f8f07fef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.864+0000 7f8f07fef640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-4 info.flags=0x0 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.864+0000 7f8f07fef640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-4 to cache LRU end 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.864+0000 7f8f07fef640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-4 : miss 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.864+0000 7f8f07fef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.864+0000 7f8f07fef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.864+0000 7f8f07fef640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-4 info.flags=0x0 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.864+0000 7f8f07fef640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-4 to cache LRU end 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.864+0000 7f8f07fef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5): adding source pipe: {s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.864+0000 7f8f07fef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5): adding dest pipe: {s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.864+0000 7f8f07fef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.864+0000 7f8f07fef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.864+0000 7f8f07fef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.864+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e19705be0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.864+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e19705be0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.864+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e19705be0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.865+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e19705be0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.865+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.865+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e196d71b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.865+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.865+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.865+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e196da040:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.865+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e196da040:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.865+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.865+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.865+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.865+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.865+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.865+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.865+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 21d616e84124bb2fd9df550b8d28f677a052e2e9596ba21d1d85859c61cf49d5 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.865+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:21d616e84124bb2fd9df550b8d28f677a052e2e9596ba21d1d85859c61cf49d5 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.865+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.865+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.865+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.865+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.866 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.865+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = efd3989678a384ae2ec68357d54ba7a35cd70e820554ea8b938a93ef0ef4417d 2026-03-20T11:45:37.866 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.865+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=efd3989678a384ae2ec68357d54ba7a35cd70e820554ea8b938a93ef0ef4417d 2026-03-20T11:45:37.866 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.865+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.866 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.865+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.866 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.865+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.866 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.865+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=51, curl_handle=0x557e196b6260 2026-03-20T11:45:37.866 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.865+0000 7f8f59593900 20 run: stack=0x557e196c5e20 is io blocked 2026-03-20T11:45:37.866 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.865+0000 7f8eb2ffd640 20 link_request req_data=0x7f8ee000f390 req_data->id=51, curl_handle=0x557e196b6260 2026-03-20T11:45:37.866 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:37.866+0000 7fd03bac5640 -1 req 2024226790977789349 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:37.884 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f027e4640 20 rgw rados thread: process(): notifying mdlog change, shard_id=0 2026-03-20T11:45:37.884 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f027e4640 20 rgw rados thread: cr:s=0x7f8f480e9960:op=0x7f8f480bb8e0:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:45:37.884 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f027e4640 20 rgw rados thread: cr:s=0x7f8f480e9960:op=0x7f8f480bb8e0:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:45:37.884 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f027e4640 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.884 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f027e4640 20 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.884 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f027e4640 20 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.884 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f027e4640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.884 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.884 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:d0bca111f8628137adc4c16f123496dcdd1d590d06cb5d9acd68b39fe656fb97 2026-03-20T11:45:37.884 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.884 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.884 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f027e4640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]payload request hash = d0bca111f8628137adc4c16f123496dcdd1d590d06cb5d9acd68b39fe656fb97 2026-03-20T11:45:37.884 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f027e4640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]canonical request = POST 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:notify=&rgwx-zonegroup=a&type=metadata 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:d0bca111f8628137adc4c16f123496dcdd1d590d06cb5d9acd68b39fe656fb97 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:d0bca111f8628137adc4c16f123496dcdd1d590d06cb5d9acd68b39fe656fb97 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f027e4640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]canonical request hash = fe749b28ed4260d71c4b587fde574950087017d84b090187a576a1453cced571 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f027e4640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:fe749b28ed4260d71c4b587fde574950087017d84b090187a576a1453cced571 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f027e4640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f027e4640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f027e4640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f027e4640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f027e4640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]generated signature = bfb6fcc10aa95eec7246861cf08e7df33b8f5a94e4dc27679996bb3ee376ce75 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f027e4640 20 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=bfb6fcc10aa95eec7246861cf08e7df33b8f5a94e4dc27679996bb3ee376ce75 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f027e4640 20 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: d0bca111f8628137adc4c16f123496dcdd1d590d06cb5d9acd68b39fe656fb97 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f027e4640 20 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f027e4640 20 sending request to http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f027e4640 20 register_request mgr=0x557e192008f8 req_data->id=1, curl_handle=0x7f8f48098c10 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f027e4640 20 run: stack=0x7f8f480e9960 is io blocked 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f02fe5640 20 link_request req_data=0x7f8f480bb490 req_data->id=1, curl_handle=0x7f8f48098c10 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f027e4640 20 rgw rados thread: cr:s=0x7f8f480e9960:op=0x7f8f480bb8e0:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f027e4640 20 rgw rados thread: cr:s=0x7f8f480e9960:op=0x7f8f480bb8e0:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f027e4640 20 rgw rados thread: cr:s=0x7f8f480e9960:op=0x7f8f480bb8e0:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f027e4640 20 rgw rados thread: cr:s=0x7f8f480e9960:op=0x7f8f480bb8e0:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f027e4640 15 stack 0x7f8f480e9960 end 2026-03-20T11:45:37.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.887+0000 7f8f027e4640 20 run: stack=0x7f8f480e9960 is done 2026-03-20T11:45:37.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e196c0080:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e196c0080:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:37.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e1920c920:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f59593900 15 stack 0x557e19686770 end 2026-03-20T11:45:37.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f59593900 20 run: stack=0x557e19686770 is done 2026-03-20T11:45:37.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f59593900 20 collect(): s=0x557e196d8570 stack=0x557e19686770 is complete 2026-03-20T11:45:37.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f59593900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f59593900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e1967c360:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f59593900 15 stack 0x557e196d8570 end 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f59593900 20 stack->operate() returned ret=-22 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f59593900 20 run: stack=0x557e196d8570 is done 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f59593900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f59593900 4 starting trim on bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e1970dc80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e1970dc80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f59593900 20 enqueued request req=0x557e196b1c70 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f59593900 20 req: 0x557e196b1c70 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f077ee640 20 dequeued request req=0x557e196b1c70 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f077ee640 20 RGWWQ: empty 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f077ee640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 : miss 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f077ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f59593900 20 run: stack=0x557e196d8570 is io blocked 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f077ee640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f077ee640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 info.flags=0x17 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f077ee640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 to cache LRU end 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f077ee640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f077ee640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f077ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f077ee640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-53 : miss 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f077ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f077ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f077ee640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-53 info.flags=0x0 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f077ee640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-53 to cache LRU end 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f077ee640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-53 : miss 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.899+0000 7f8f077ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f077ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f077ee640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-53 info.flags=0x0 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f077ee640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-53 to cache LRU end 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f077ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13): adding source pipe: {s={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f077ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13): adding dest pipe: {s={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f077ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f077ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f077ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e1970dc80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e1970dc80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e1970dc80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e1970dc80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e196c0060:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 20 run: stack=0x557e196d8570 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e196d4ad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e196d4ad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.899 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = b258759506da7b79ad86c5f46297c8fd11fa2072987f36ff245c44c2e3a8c39d 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr:b258759506da7b79ad86c5f46297c8fd11fa2072987f36ff245c44c2e3a8c39d 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 07185289c0f3e82ad1b3f7bbaad5daa685c6a3d4ae0dd2dd766df4eb0cba5d53 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=07185289c0f3e82ad1b3f7bbaad5daa685c6a3d4ae0dd2dd766df4eb0cba5d53 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=52, curl_handle=0x557e19681800 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8f59593900 20 run: stack=0x557e19686770 is io blocked 2026-03-20T11:45:37.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.900+0000 7f8eb2ffd640 20 link_request req_data=0x557e196823d0 req_data->id=52, curl_handle=0x557e19681800 2026-03-20T11:45:37.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e19700a10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e19700a10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e19700a10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e19700a10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c4c90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 15 stack 0x557e196700a0 end 2026-03-20T11:45:37.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 run: stack=0x557e196700a0 is done 2026-03-20T11:45:37.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 collect(): s=0x557e196c4280 stack=0x557e196700a0 is complete 2026-03-20T11:45:37.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 10 min_generation is 1 2026-03-20T11:45:37.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196ff680:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.0 2026-03-20T11:45:37.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.1 2026-03-20T11:45:37.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.2 2026-03-20T11:45:37.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.3 2026-03-20T11:45:37.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.4 2026-03-20T11:45:37.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.5 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.6 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.7 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.8 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.9 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.10 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 run: stack=0x557e196c4280 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c44d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196c4c90:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 cr:s=0x557e196d4990:op=0x557e19700a10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e1920e460:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e192c61f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1966f1f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e1920a900:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 cr:s=0x557e1920ea50:op=0x557e19705800:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 cr:s=0x557e19212060:op=0x557e19705df0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 cr:s=0x557e196d6350:op=0x557e196ffd00:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 cr:s=0x557e197018e0:op=0x557e197002f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c44d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 run: stack=0x557e196700a0 is io blocked 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196c4c90:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 run: stack=0x557e196c3030 is io blocked 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 cr:s=0x557e196d4990:op=0x557e19700a10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 run: stack=0x557e196d4990 is io blocked 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e1920e460:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 run: stack=0x557e196bcde0 is io blocked 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e192c61f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 run: stack=0x557e19701570 is io blocked 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.905+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1966f1f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.906+0000 7f8f59593900 20 run: stack=0x557e19688770 is io blocked 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.906+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e1920a900:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.906+0000 7f8f59593900 20 run: stack=0x557e196baa80 is io blocked 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.906+0000 7f8f59593900 20 cr:s=0x557e1920ea50:op=0x557e19705800:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.906+0000 7f8f59593900 20 run: stack=0x557e1920ea50 is io blocked 2026-03-20T11:45:37.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.906+0000 7f8f59593900 20 cr:s=0x557e19212060:op=0x557e19705df0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.906+0000 7f8f59593900 20 run: stack=0x557e19212060 is io blocked 2026-03-20T11:45:37.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.906+0000 7f8f59593900 20 cr:s=0x557e196d6350:op=0x557e196ffd00:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.906+0000 7f8f59593900 20 run: stack=0x557e196d6350 is io blocked 2026-03-20T11:45:37.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.906+0000 7f8f59593900 20 cr:s=0x557e197018e0:op=0x557e197002f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 run: stack=0x557e197018e0 is io blocked 2026-03-20T11:45:37.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196c4c90:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196d4990:op=0x557e19700a10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c44d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e1920e460:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e192c61f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1966f1f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e19212060:op=0x557e19705df0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e1920a900:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196c4c90:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196d4990:op=0x557e19700a10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c44d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e1920ea50:op=0x557e19705800:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920ff20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e196da040:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e196da040:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196d6350:op=0x557e196ffd00:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e1920e460:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e192c61f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1966f1f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e19212060:op=0x557e19705df0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e1920a900:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196c4c90:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196d4990:op=0x557e19700a10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c44d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e1920ea50:op=0x557e19705800:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920ff20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e196d71b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 15 stack 0x557e196c5e20 end 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 run: stack=0x557e196c5e20 is done 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196d6350:op=0x557e196ffd00:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e1920e460:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e192c61f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1966f1f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e19212060:op=0x557e19705df0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e1920a900:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196c4c90:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 15 stack 0x557e196c3030 end 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 run: stack=0x557e196c3030 is done 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196d4990:op=0x557e19700a10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 15 stack 0x557e196d4990 end 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 run: stack=0x557e196d4990 is done 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e196c44d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 15 stack 0x557e196700a0 end 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 run: stack=0x557e196700a0 is done 2026-03-20T11:45:37.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e1920ea50:op=0x557e19705800:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920ff20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 collect(): s=0x557e196c7af0 stack=0x557e196c5e20 is complete 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 15 stack 0x557e196c7af0 end 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 stack->operate() returned ret=-22 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is done 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196d6350:op=0x557e196ffd00:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e1920e460:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 15 stack 0x557e196bcde0 end 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 run: stack=0x557e196bcde0 is done 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e192c61f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 15 stack 0x557e19701570 end 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 run: stack=0x557e19701570 is done 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1966f1f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 15 stack 0x557e19688770 end 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 run: stack=0x557e19688770 is done 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e19212060:op=0x557e19705df0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 15 stack 0x557e19212060 end 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 run: stack=0x557e19212060 is done 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196baa80:op=0x557e1920a900:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 15 stack 0x557e196baa80 end 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 run: stack=0x557e196baa80 is done 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196ff680:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e1920ea50:op=0x557e19705800:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 15 stack 0x557e1920ea50 end 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 run: stack=0x557e1920ea50 is done 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920ff20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196d6350:op=0x557e196ffd00:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 15 stack 0x557e196d6350 end 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 run: stack=0x557e196d6350 is done 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196ff680:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196b95b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 15 stack 0x557e196be570 end 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 run: stack=0x557e196be570 is done 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 4 starting trim on bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196ff680:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19210960:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 collect(): s=0x557e196ad300 stack=0x557e196be570 is complete 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-19[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24]) markers=[,,,,,,,,00000000003.27.6,,], shards=11 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196cffb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196ff680:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196ffd00:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 10 trimming bilog shard 8 of :dipepu-19[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24]) at marker 00000000003.27.6 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196cffb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 enqueued request req=0x557e19702960 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 req: 0x557e19702960 2026-03-20T11:45:37.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f06fed640 20 dequeued request req=0x557e19702960 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f06fed640 20 RGWWQ: empty 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f06fed640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 : miss 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f06fed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is io blocked 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196ff680:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e192c61f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e197018e0:op=0x557e197002f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196ff680:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e192c61f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24.8 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 run: stack=0x557e196bcde0 is io blocked 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e197018e0:op=0x557e197002f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.908+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196ff680:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f59593900 20 cr:s=0x557e197018e0:op=0x557e197002f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196ff680:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f59593900 20 cr:s=0x557e197018e0:op=0x557e197002f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f59593900 15 stack 0x557e197018e0 end 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f59593900 20 run: stack=0x557e197018e0 is done 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196ff680:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196ff680:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196ff680:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196ff680:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196ff680:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f59593900 20 enqueued request req=0x557e19702830 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f59593900 20 req: 0x557e19702830 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f067ec640 20 dequeued request req=0x557e19702830 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f067ec640 20 RGWWQ: empty 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f06fed640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f06fed640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 info.flags=0x17 2026-03-20T11:45:37.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f06fed640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 to cache LRU end 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f06fed640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f06fed640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f06fed640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f06fed640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-14 : miss 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f06fed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f59593900 20 run: stack=0x557e196c4280 is io blocked 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f06fed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f06fed640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-14 info.flags=0x0 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f06fed640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-14 to cache LRU end 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f06fed640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-14 : miss 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.909+0000 7f8f06fed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f06fed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f06fed640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-14 info.flags=0x0 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f06fed640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-14 to cache LRU end 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f06fed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19): adding source pipe: {s={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f06fed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19): adding dest pipe: {s={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f06fed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f06fed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f06fed640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e192c61f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e192c61f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196cffb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e192c61f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196cffb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 cr:s=0x557e196bcde0:op=0x557e192c61f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 15 stack 0x557e196bcde0 end 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 run: stack=0x557e196bcde0 is done 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196cffb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196ffd00:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196cffb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19210960:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 15 stack 0x557e196ad300 end 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 run: stack=0x557e196ad300 is done 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196b95b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19708b20:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 4 starting trim on bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1920ff20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196cffb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1920ff20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.909 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 4e4d97df3e05bc1a8b144ab82331312aaa653ab7b2fafd1b030a17cd23648b13 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:4e4d97df3e05bc1a8b144ab82331312aaa653ab7b2fafd1b030a17cd23648b13 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 6aae7a287077f696472b473ff29bb4c2c60329b66916d7acf0c8f0ee2987c62b 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=6aae7a287077f696472b473ff29bb4c2c60329b66916d7acf0c8f0ee2987c62b 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=53, curl_handle=0x557e196b6260 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 run: stack=0x557e19688770 is io blocked 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196cffb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 enqueued request req=0x557e19671360 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 req: 0x557e19671360 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8eb2ffd640 20 link_request req_data=0x7f8ee000f390 req_data->id=53, curl_handle=0x557e196b6260 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f05feb640 20 dequeued request req=0x557e19671360 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f05feb640 20 RGWWQ: empty 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f05feb640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 : miss 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f05feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f59593900 20 run: stack=0x557e196ad300 is io blocked 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f067ec640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 info.flags=0x11 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f067ec640 10 rgw async rados processor: moving a1.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 to cache LRU end 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f067ec640 10 rgw async rados processor: distributing notification oid=a1.rgw.control:notify.5 cni=[op: 0, obj: a1.rgw.meta:root:.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10, ofs0, ns] 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f05feb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f05feb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 info.flags=0x17 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f05feb640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 to cache LRU end 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f05feb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f05feb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f05feb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f05feb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-36 : miss 2026-03-20T11:45:37.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f05feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f05feb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f05feb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-36 info.flags=0x0 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f05feb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-36 to cache LRU end 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f05feb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-36 : miss 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.910+0000 7f8f05feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f30ff9640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 335007449321 cookie 94000080666336 notifier 6226 bl.length()=565 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f30ff9640 10 rgw watcher librados: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 info.flags=0x11 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f30ff9640 10 rgw watcher librados: moving a1.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 to cache LRU end 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f05feb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f05feb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-36 info.flags=0x0 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f05feb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-36 to cache LRU end 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f05feb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41): adding source pipe: {s={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f05feb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41): adding dest pipe: {s={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f05feb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f05feb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f05feb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196cffb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196cffb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196cffb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196cffb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19708b20:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e196c4c90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19708b20:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e19700a10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e19700a10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 08933345792fb3a4ecd8ce6ea4b73f40cabf98416f74c19e36e375e98247e797 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:08933345792fb3a4ecd8ce6ea4b73f40cabf98416f74c19e36e375e98247e797 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = db1a755866c6cc3646f58adcbeb79a19e4aaf7dad403cfcbcfaf0ee290ded480 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=db1a755866c6cc3646f58adcbeb79a19e4aaf7dad403cfcbcfaf0ee290ded480 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=54, curl_handle=0x557e196861c0 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8f59593900 20 run: stack=0x557e19701570 is io blocked 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.911+0000 7f8eb2ffd640 20 link_request req_data=0x557e196b2870 req_data->id=54, curl_handle=0x557e196861c0 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.913+0000 7f8f067ec640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-5[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10]): orig_sources= new_sources= 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.913+0000 7f8f067ec640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-5[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10]): potential sources added=[] removed=[] 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.913+0000 7f8f067ec640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-5[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10]): orig_dests= new_dests= 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.913+0000 7f8f067ec640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-5[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10]): potential dests added=[] removed=[] 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.913+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196ff680:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.913+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196ff680:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.913+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196ff680:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.913+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196ff680:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.913+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.913+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196da040:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.913+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196da040:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.913+0000 7f8f59593900 20 enqueued request req=0x557e191a2af0 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.913+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.913+0000 7f8f59593900 20 req: 0x557e191a2af0 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.913+0000 7f8f057ea640 20 dequeued request req=0x557e191a2af0 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.913+0000 7f8f057ea640 20 RGWWQ: empty 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.913+0000 7f8f59593900 20 run: stack=0x557e196c4280 is io blocked 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f057ea640 10 rgw async rados processor: removing a1.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 from cache 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f057ea640 10 rgw async rados processor: distributing notification oid=a1.rgw.control:notify.5 cni=[op: 1, obj: a1.rgw.meta:root:.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10, ofs0, ns] 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f23fff640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 335007449322 cookie 94000080666336 notifier 6226 bl.length()=205 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196da040:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196da040:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196da040:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196da040:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f59593900 15 stack 0x557e196c4280 end 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f59593900 20 run: stack=0x557e196c4280 is done 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f59593900 4 starting trim on bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196cffb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196cffb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f59593900 20 enqueued request req=0x557e1966eb20 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f59593900 20 req: 0x557e1966eb20 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f04fe9640 20 dequeued request req=0x557e1966eb20 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f04fe9640 20 RGWWQ: empty 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f04fe9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 : miss 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f04fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f59593900 20 run: stack=0x557e196c4280 is io blocked 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f04fe9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f04fe9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 info.flags=0x17 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f04fe9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 to cache LRU end 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f04fe9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f04fe9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f04fe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f04fe9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-22 : miss 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.914+0000 7f8f04fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f04fe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f04fe9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-22 info.flags=0x0 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f04fe9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-22 to cache LRU end 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f04fe9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-22 : miss 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f04fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f04fe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f04fe9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-22 info.flags=0x0 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f04fe9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-22 to cache LRU end 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f04fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27): adding source pipe: {s={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f04fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27): adding dest pipe: {s={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f04fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f04fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f04fe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196cffb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196cffb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196cffb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196cffb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196da040:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 20 run: stack=0x557e196c4280 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920bb30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920bb30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 7bb2393468106f43572de73065e8d31ced3813a82f9c5f354ac625ecb6ccc55c 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:7bb2393468106f43572de73065e8d31ced3813a82f9c5f354ac625ecb6ccc55c 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 581404e4dd2d373e17cc09f914881d36f1e07536c78b8075b19fe4fec0445da9 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=581404e4dd2d373e17cc09f914881d36f1e07536c78b8075b19fe4fec0445da9 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=55, curl_handle=0x557e196b2160 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8f59593900 20 run: stack=0x557e196be570 is io blocked 2026-03-20T11:45:37.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.915+0000 7f8eb2ffd640 20 link_request req_data=0x7f8ec0025770 req_data->id=55, curl_handle=0x557e196b2160 2026-03-20T11:45:37.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e196d4ad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e196d4ad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e196d4ad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e196d4ad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e196c0060:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 15 stack 0x557e19686770 end 2026-03-20T11:45:37.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 run: stack=0x557e19686770 is done 2026-03-20T11:45:37.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 collect(): s=0x557e196d8570 stack=0x557e19686770 is complete 2026-03-20T11:45:37.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) markers=[00000000004.65.1,00000000002.83.1,00000000004.51.1,00000000002.21.1,00000000004.74.1,00000000002.72.1,00000000002.84.1,00000000004.73.1,00000000002.22.1,00000000002.50.1,00000000002.35.1], shards=11 2026-03-20T11:45:37.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196c0060:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 10 trimming bilog shard 0 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.65.1 2026-03-20T11:45:37.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 10 trimming bilog shard 1 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.83.1 2026-03-20T11:45:37.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 10 trimming bilog shard 2 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.51.1 2026-03-20T11:45:37.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 10 trimming bilog shard 3 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.21.1 2026-03-20T11:45:37.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 10 trimming bilog shard 4 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.74.1 2026-03-20T11:45:37.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 10 trimming bilog shard 5 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.72.1 2026-03-20T11:45:37.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 10 trimming bilog shard 6 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.84.1 2026-03-20T11:45:37.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 10 trimming bilog shard 7 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.73.1 2026-03-20T11:45:37.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 10 trimming bilog shard 8 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.22.1 2026-03-20T11:45:37.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 10 trimming bilog shard 9 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.50.1 2026-03-20T11:45:37.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 10 trimming bilog shard 10 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.35.1 2026-03-20T11:45:37.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 run: stack=0x557e196d8570 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e1966f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e196d4990:op=0x557e19707ab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196d4ad0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e197017e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e1920a900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e196d8aa0:op=0x557e1970a470:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e196d4320:op=0x557e196ffd00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e196701e0:op=0x557e1970c280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e192c8610:op=0x557e19211f70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e196fe670:op=0x557e19212770:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e196bc070:op=0x557e196d0770:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e1966f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.0 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 run: stack=0x557e19686770 is io blocked 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e196d4990:op=0x557e19707ab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.1 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 run: stack=0x557e196d4990 is io blocked 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196d4ad0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.2 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 run: stack=0x557e196c3030 is io blocked 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e197017e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.3 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 run: stack=0x557e196700a0 is io blocked 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e1920a900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.4 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 run: stack=0x557e196c5e20 is io blocked 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e196d8aa0:op=0x557e1970a470:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.5 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 run: stack=0x557e196d8aa0 is io blocked 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e196d4320:op=0x557e196ffd00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.6 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 run: stack=0x557e196d4320 is io blocked 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e196701e0:op=0x557e1970c280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.7 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 run: stack=0x557e196701e0 is io blocked 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e192c8610:op=0x557e19211f70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.8 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 run: stack=0x557e192c8610 is io blocked 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e196fe670:op=0x557e19212770:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.9 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 run: stack=0x557e196fe670 is io blocked 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 cr:s=0x557e196bc070:op=0x557e196d0770:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.10 2026-03-20T11:45:37.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.943+0000 7f8f59593900 20 run: stack=0x557e196bc070 is io blocked 2026-03-20T11:45:37.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.953+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1920ff20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.953+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1920ff20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.953+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1920ff20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.953+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e1920ff20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.953+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e196b95b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.953+0000 7f8f59593900 15 stack 0x557e19688770 end 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.953+0000 7f8f59593900 20 run: stack=0x557e19688770 is done 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.953+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.953+0000 7f8f59593900 20 collect(): s=0x557e196c7af0 stack=0x557e19688770 is complete 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.953+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.953+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) markers=[,,,,,00000000002.32.11,,,00000000003.22.6,00000000011.52.6,], shards=11 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.953+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e19726630:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.953+0000 7f8f59593900 10 trimming bilog shard 5 of :dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) at marker 00000000002.32.11 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.953+0000 7f8f59593900 10 trimming bilog shard 8 of :dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) at marker 00000000003.22.6 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.953+0000 7f8f59593900 10 trimming bilog shard 9 of :dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) at marker 00000000011.52.6 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.953+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.953+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e192c61f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.953+0000 7f8f59593900 20 cr:s=0x557e19722830:op=0x557e19722030:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.953+0000 7f8f59593900 20 cr:s=0x557e196d1010:op=0x557e197282f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.953+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e192c61f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.953+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19.5 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.953+0000 7f8f59593900 20 run: stack=0x557e19688770 is io blocked 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.953+0000 7f8f59593900 20 cr:s=0x557e19722830:op=0x557e19722030:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.953+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19.8 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.953+0000 7f8f59593900 20 run: stack=0x557e19722830 is io blocked 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.953+0000 7f8f59593900 20 cr:s=0x557e196d1010:op=0x557e197282f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.953+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19.9 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.953+0000 7f8f59593900 20 run: stack=0x557e196d1010 is io blocked 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.954+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e19700a10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.954+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e19700a10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.954+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e19700a10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.954+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e19700a10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.954+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e196c4c90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.954+0000 7f8f59593900 15 stack 0x557e19701570 end 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.954+0000 7f8f59593900 20 run: stack=0x557e19701570 is done 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.954+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19708b20:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.954+0000 7f8f59593900 20 collect(): s=0x557e196ad300 stack=0x557e19701570 is complete 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.954+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.954+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-36[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41]) markers=[,,,,,,,,,,00000000001.34.6], shards=11 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.954+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196c4c90:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.954+0000 7f8f59593900 10 trimming bilog shard 10 of :dipepu-36[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41]) at marker 00000000001.34.6 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.954+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.954+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1920c700:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.954+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1920c700:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.954+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41.10 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.954+0000 7f8f59593900 20 run: stack=0x557e19701570 is io blocked 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.958+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920bb30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.958+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920bb30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.958+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920bb30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.958+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920bb30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.958+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196da040:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.958+0000 7f8f59593900 15 stack 0x557e196be570 end 2026-03-20T11:45:37.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.958+0000 7f8f59593900 20 run: stack=0x557e196be570 is done 2026-03-20T11:45:37.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.958+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.958+0000 7f8f59593900 20 collect(): s=0x557e196c4280 stack=0x557e196be570 is complete 2026-03-20T11:45:37.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.958+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.958+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-22[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27]) markers=[,,,,,,,,00000000006.25.9,,], shards=11 2026-03-20T11:45:37.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.958+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19726df0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.958+0000 7f8f59593900 10 trimming bilog shard 8 of :dipepu-22[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27]) at marker 00000000006.25.9 2026-03-20T11:45:37.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.958+0000 7f8f59593900 20 run: stack=0x557e196c4280 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.958+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e19724370:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.958+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e19724370:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.958+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27.8 2026-03-20T11:45:37.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.958+0000 7f8f59593900 20 run: stack=0x557e196be570 is io blocked 2026-03-20T11:45:37.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.978+0000 7f8f59593900 20 cr:s=0x557e196fe670:op=0x557e19212770:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.978+0000 7f8f59593900 20 cr:s=0x557e196fe670:op=0x557e19212770:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.978+0000 7f8f59593900 20 cr:s=0x557e196fe670:op=0x557e19212770:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.978+0000 7f8f59593900 20 cr:s=0x557e196fe670:op=0x557e19212770:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.978+0000 7f8f59593900 15 stack 0x557e196fe670 end 2026-03-20T11:45:37.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.978+0000 7f8f59593900 20 run: stack=0x557e196fe670 is done 2026-03-20T11:45:37.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.978+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196c0060:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.978+0000 7f8f59593900 20 run: stack=0x557e196d8570 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e1966f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e1966f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e1966f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e1966f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 15 stack 0x557e19686770 end 2026-03-20T11:45:37.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 run: stack=0x557e19686770 is done 2026-03-20T11:45:37.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196c0060:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 run: stack=0x557e196d8570 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196d4ad0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196d4ad0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196d4ad0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196d4ad0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 15 stack 0x557e196c3030 end 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 run: stack=0x557e196c3030 is done 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196c0060:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 run: stack=0x557e196d8570 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1920c700:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1920c700:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e196bc070:op=0x557e196d0770:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e196d1010:op=0x557e197282f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1920c700:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e196bc070:op=0x557e196d0770:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e196d1010:op=0x557e197282f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e1920c700:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 15 stack 0x557e19701570 end 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 run: stack=0x557e19701570 is done 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e196bc070:op=0x557e196d0770:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e196d1010:op=0x557e197282f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e196c4c90:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e196bc070:op=0x557e196d0770:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 15 stack 0x557e196bc070 end 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 run: stack=0x557e196bc070 is done 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e196d1010:op=0x557e197282f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 15 stack 0x557e196d1010 end 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 run: stack=0x557e196d1010 is done 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19708b20:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 15 stack 0x557e196ad300 end 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 run: stack=0x557e196ad300 is done 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196c0060:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 run: stack=0x557e196d8570 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e1920a900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e19726630:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e1920a900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19706760:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 4 starting trim on bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e1920a900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e197282f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e1920a900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 15 stack 0x557e196c5e20 end 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 run: stack=0x557e196c5e20 is done 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e197282f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 enqueued request req=0x557e192c8310 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 req: 0x557e192c8310 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f20ff9640 20 dequeued request req=0x557e192c8310 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f20ff9640 20 RGWWQ: empty 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f20ff9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 : miss 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f20ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 run: stack=0x557e196ad300 is io blocked 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196c0060:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.984+0000 7f8f59593900 20 run: stack=0x557e196d8570 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e19724370:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e19724370:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e19724370:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e19724370:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 15 stack 0x557e196be570 end 2026-03-20T11:45:37.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 run: stack=0x557e196be570 is done 2026-03-20T11:45:37.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19726df0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:45:37.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 15 stack 0x557e196c4280 end 2026-03-20T11:45:37.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 run: stack=0x557e196c4280 is done 2026-03-20T11:45:37.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 4 starting trim on bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:45:37.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920a900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920a900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 enqueued request req=0x557e196b9c50 2026-03-20T11:45:37.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 req: 0x557e196b9c50 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 run: stack=0x557e196be570 is io blocked 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e197017e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e197017e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e197017e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e197017e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 15 stack 0x557e196700a0 end 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 run: stack=0x557e196700a0 is done 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196c0060:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 run: stack=0x557e196d8570 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196d8aa0:op=0x557e1970a470:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196d8aa0:op=0x557e1970a470:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196701e0:op=0x557e1970c280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e192c61f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196d4990:op=0x557e19707ab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196d8aa0:op=0x557e1970a470:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196701e0:op=0x557e1970c280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e192c61f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196d4990:op=0x557e19707ab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196d8aa0:op=0x557e1970a470:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 15 stack 0x557e196d8aa0 end 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 run: stack=0x557e196d8aa0 is done 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196701e0:op=0x557e1970c280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e192c61f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196d4990:op=0x557e19707ab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196c0060:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 run: stack=0x557e196d8570 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196d4320:op=0x557e196ffd00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196701e0:op=0x557e1970c280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 15 stack 0x557e196701e0 end 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f047e8640 20 dequeued request req=0x557e196b9c50 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 run: stack=0x557e196701e0 is done 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f047e8640 20 RGWWQ: empty 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e19688770:op=0x557e192c61f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 15 stack 0x557e19688770 end 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 run: stack=0x557e19688770 is done 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196d4990:op=0x557e19707ab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 15 stack 0x557e196d4990 end 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 run: stack=0x557e196d4990 is done 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196d4320:op=0x557e196ffd00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196c0060:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e19726630:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196d4320:op=0x557e196ffd00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196c0060:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 run: stack=0x557e196d8570 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196d4320:op=0x557e196ffd00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 15 stack 0x557e196d4320 end 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f047e8640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 : miss 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 run: stack=0x557e196d4320 is done 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196c0060:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 run: stack=0x557e196d8570 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f047e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f20ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f20ff9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 info.flags=0x17 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f20ff9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 to cache LRU end 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f20ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f20ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f20ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:45:37.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f20ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-25 : miss 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f20ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e192c8610:op=0x557e19211f70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e192c8610:op=0x557e19211f70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e19722830:op=0x557e19722030:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e192c8610:op=0x557e19211f70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e19722830:op=0x557e19722030:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e192c8610:op=0x557e19211f70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 15 stack 0x557e192c8610 end 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 run: stack=0x557e192c8610 is done 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e19722830:op=0x557e19722030:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e196c0060:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e19722830:op=0x557e19722030:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 15 stack 0x557e19722830 end 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 run: stack=0x557e19722830 is done 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196d8570:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 15 stack 0x557e196d8570 end 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 run: stack=0x557e196d8570 is done 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e19726630:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 15 stack 0x557e196c7af0 end 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is done 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 4 starting trim on bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e192c61f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 4 starting trim on bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e192c61f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 enqueued request req=0x557e19671360 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 req: 0x557e19671360 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f167fc640 20 dequeued request req=0x557e19671360 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f167fc640 20 RGWWQ: empty 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f167fc640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 : miss 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f167fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 run: stack=0x557e196700a0 is io blocked 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1970a420:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1970a420:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 enqueued request req=0x557e19577a40 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 RGWWQ: 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 req: 0x557e19577a40 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is io blocked 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f20ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f20ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-25 info.flags=0x0 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f20ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-25 to cache LRU end 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f20ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-25 : miss 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.985+0000 7f8f20ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f177fe640 20 dequeued request req=0x557e19577a40 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f177fe640 20 RGWWQ: empty 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f177fe640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 : miss 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f177fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f047e8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f047e8640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 info.flags=0x17 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f047e8640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 to cache LRU end 2026-03-20T11:45:37.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f047e8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f047e8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f047e8640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f047e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f047e8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-35 : miss 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f047e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f20ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f20ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-25 info.flags=0x0 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f20ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-25 to cache LRU end 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f20ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30): adding source pipe: {s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f20ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30): adding dest pipe: {s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f20ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f20ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f20ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e197282f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e197282f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e197282f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e197282f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19706760:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e196b95b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19706760:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 20 run: stack=0x557e196ad300 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e196d0000:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e196d0000:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 9eccd3da69f5c3239bc5a0e083e32f5e4c6626375042f0d605aac0c4b4f48b4c 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:9eccd3da69f5c3239bc5a0e083e32f5e4c6626375042f0d605aac0c4b4f48b4c 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 27ece89ac0c156e1b2b914f9a60f748fe4a0b4c3c01e724d1b16205a72d5d39c 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=27ece89ac0c156e1b2b914f9a60f748fe4a0b4c3c01e724d1b16205a72d5d39c 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=56, curl_handle=0x557e196b2160 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8eb2ffd640 20 link_request req_data=0x7f8ec0025770 req_data->id=56, curl_handle=0x557e196b2160 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f047e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f047e8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-35 info.flags=0x0 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f047e8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-35 to cache LRU end 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f177fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f047e8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-35 : miss 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f177fe640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 info.flags=0x17 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f047e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f177fe640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 to cache LRU end 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f177fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f177fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f177fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f177fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-51 : miss 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f177fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f167fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f167fc640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 info.flags=0x17 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f167fc640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 to cache LRU end 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f167fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f167fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f167fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f167fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-26 : miss 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f167fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 20 run: stack=0x557e196c5e20 is io blocked 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f177fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f177fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-51 info.flags=0x0 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f177fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-51 to cache LRU end 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f177fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-51 : miss 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f177fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f177fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f177fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-51 info.flags=0x0 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f177fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-51 to cache LRU end 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f177fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11): adding source pipe: {s={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f177fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11): adding dest pipe: {s={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f047e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f047e8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-35 info.flags=0x0 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f047e8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-35 to cache LRU end 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f177fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f177fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f047e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40): adding source pipe: {s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f047e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40): adding dest pipe: {s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f167fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f177fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f167fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-26 info.flags=0x0 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f167fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-26 to cache LRU end 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f167fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-26 : miss 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f167fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f047e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f047e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1970a420:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1970a420:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1970a420:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1970a420:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.986+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 cr:s=0x557e196bc070:op=0x557e196c4c90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 cr:s=0x557e196bc070:op=0x557e1920ff20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 cr:s=0x557e196bc070:op=0x557e1920ff20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f047e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 1fe51572df55b2c96f63bd9d6b3f622f0772f33f4e0da268101b38c8ed7b5ac9 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:1fe51572df55b2c96f63bd9d6b3f622f0772f33f4e0da268101b38c8ed7b5ac9 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 26b260402b0f00cd6dfcf92100bad28085ce4b67a49fd6abc27a87578b588df2 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=26b260402b0f00cd6dfcf92100bad28085ce4b67a49fd6abc27a87578b588df2 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=57, curl_handle=0x557e196861c0 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8eb2ffd640 20 link_request req_data=0x557e196b2870 req_data->id=57, curl_handle=0x557e196861c0 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f167fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f167fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-26 info.flags=0x0 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f167fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-26 to cache LRU end 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f167fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31): adding source pipe: {s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f167fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31): adding dest pipe: {s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f167fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f167fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f167fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 run: stack=0x557e196bc070 is io blocked 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920a900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e192c61f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920a900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e192c61f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920a900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e192c61f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e1920a900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e192c61f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e19700a10:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 run: stack=0x557e196be570 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1920e270:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 run: stack=0x557e196700a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e19705d20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196d8e00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e19705d20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = fe9de608be5a886ba184702ce9ec6923777189f6ed5f7545b7d93aae09046a03 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:fe9de608be5a886ba184702ce9ec6923777189f6ed5f7545b7d93aae09046a03 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 7e813bbe76a0d8965e73264597e8d019a695f9f1f48fc20ed0ba03cdf8a5e5a2 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=7e813bbe76a0d8965e73264597e8d019a695f9f1f48fc20ed0ba03cdf8a5e5a2 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=58, curl_handle=0x557e196b6260 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8eb2ffd640 20 link_request req_data=0x7f8ee000f390 req_data->id=58, curl_handle=0x557e196b6260 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 run: stack=0x557e19701570 is io blocked 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196d8e00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:37 +0000 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114537Z 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 966a01d889de4658ac603b64fa5c8d73fda32d91a39944ef75ea9159f9a09b5b 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:37.991 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114537Z 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:966a01d889de4658ac603b64fa5c8d73fda32d91a39944ef75ea9159f9a09b5b 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 0df570b18f855a086dc40ca0d76cec1d76a261463de99e266d08339110bf41d7 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=0df570b18f855a086dc40ca0d76cec1d76a261463de99e266d08339110bf41d7 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114537Z 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 register_request mgr=0x7ffc6aadd8f0 req_data->id=59, curl_handle=0x557e19681800 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8eb2ffd640 20 link_request req_data=0x557e196823d0 req_data->id=59, curl_handle=0x557e19681800 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.987+0000 7f8f59593900 20 run: stack=0x557e196c3030 is io blocked 2026-03-20T11:45:37.992 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:37.988+0000 7fd07833e640 -1 req 7750778611491566501 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:37.992 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:37.988+0000 7fd05a302640 -1 req 4552146274156821164 0.003000004s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:37.992 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:37.989+0000 7fd059b01640 -1 req 1740110949860655344 0.003000005s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.990+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e196d0000:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.990+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e196d0000:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.990+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e196b95b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.990+0000 7f8f59593900 15 stack 0x557e196c5e20 end 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.990+0000 7f8f59593900 20 run: stack=0x557e196c5e20 is done 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.990+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19706760:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.990+0000 7f8f59593900 20 collect(): s=0x557e196ad300 stack=0x557e196c5e20 is complete 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.990+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.990+0000 7f8f59593900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.990+0000 7f8f59593900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.990+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19706760:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.990+0000 7f8f59593900 15 stack 0x557e196ad300 end 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.990+0000 7f8f59593900 20 stack->operate() returned ret=-22 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.990+0000 7f8f59593900 20 run: stack=0x557e196ad300 is done 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.990+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.990+0000 7f8f59593900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:37.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:37.990+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:38.027 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.030+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e19705d20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:38.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.030+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e19705d20:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:38.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.030+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e19700a10:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:38.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.030+0000 7f8f59593900 15 stack 0x557e19701570 end 2026-03-20T11:45:38.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.030+0000 7f8f59593900 20 run: stack=0x557e19701570 is done 2026-03-20T11:45:38.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.030+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:38.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.030+0000 7f8f59593900 20 collect(): s=0x557e196be570 stack=0x557e19701570 is complete 2026-03-20T11:45:38.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.030+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:38.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.030+0000 7f8f59593900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:38.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.030+0000 7f8f59593900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:38.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.030+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e19679dc0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:38.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.030+0000 7f8f59593900 15 stack 0x557e196be570 end 2026-03-20T11:45:38.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.030+0000 7f8f59593900 20 stack->operate() returned ret=-22 2026-03-20T11:45:38.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.030+0000 7f8f59593900 20 run: stack=0x557e196be570 is done 2026-03-20T11:45:38.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.030+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:38.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.030+0000 7f8f59593900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:38.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.030+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:38.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196bc070:op=0x557e1920ff20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196bc070:op=0x557e1920ff20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196d8e00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e196d8e00:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196bc070:op=0x557e1920ff20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1920e270:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 15 stack 0x557e196c3030 end 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 run: stack=0x557e196c3030 is done 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196bc070:op=0x557e1920ff20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 collect(): s=0x557e196700a0 stack=0x557e196c3030 is complete 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196700a0:op=0x557e1967c360:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 15 stack 0x557e196700a0 end 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 stack->operate() returned ret=-22 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 run: stack=0x557e196700a0 is done 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196bc070:op=0x557e196c4c90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 15 stack 0x557e196bc070 end 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 run: stack=0x557e196bc070 is done 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 collect(): s=0x557e196c7af0 stack=0x557e196bc070 is complete 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 10 min_generation is 0 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 10 trimming bilogs for bucket=:dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) markers=[00000000004.41.6,00000000002.16.1,00000000004.28.6,00000000002.34.1,00000000004.45.6,00000000002.40.1,00000000002.39.1,00000000004.52.6,00000000002.73.1,00000000002.41.1,00000000002.74.1], shards=11 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196ff680:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 10 trimming bilog shard 0 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000004.41.6 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 10 trimming bilog shard 1 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.16.1 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 10 trimming bilog shard 2 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000004.28.6 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 10 trimming bilog shard 3 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.34.1 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 10 trimming bilog shard 4 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000004.45.6 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 10 trimming bilog shard 5 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.40.1 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 10 trimming bilog shard 6 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.39.1 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 10 trimming bilog shard 7 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000004.52.6 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 10 trimming bilog shard 8 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.73.1 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 10 trimming bilog shard 9 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.41.1 2026-03-20T11:45:38.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 10 trimming bilog shard 10 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.74.1 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1920a900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d3960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e192c61f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19728280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e1970d1d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e1966f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196fe670:op=0x557e197017e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196d06e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196d8aa0:op=0x557e196fff40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196b5520:op=0x557e1970c680:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196cdd00:op=0x557e196d4590:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1920a900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.0 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 run: stack=0x557e196c3030 is io blocked 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d3960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.1 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 run: stack=0x557e196be570 is io blocked 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e192c61f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.2 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 run: stack=0x557e19701570 is io blocked 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19728280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.3 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 run: stack=0x557e196ad300 is io blocked 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e1970d1d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.4 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 run: stack=0x557e196c5e20 is io blocked 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e1966f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.5 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 run: stack=0x557e19686770 is io blocked 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196fe670:op=0x557e197017e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.6 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 run: stack=0x557e196fe670 is io blocked 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196d06e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.7 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 run: stack=0x557e196c4280 is io blocked 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196d8aa0:op=0x557e196fff40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.8 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 run: stack=0x557e196d8aa0 is io blocked 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196b5520:op=0x557e1970c680:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.9 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 run: stack=0x557e196b5520 is io blocked 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 cr:s=0x557e196cdd00:op=0x557e196d4590:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.10 2026-03-20T11:45:38.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.031+0000 7f8f59593900 20 run: stack=0x557e196cdd00 is io blocked 2026-03-20T11:45:38.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19728280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19728280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19728280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196ad300:op=0x557e19728280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 15 stack 0x557e196ad300 end 2026-03-20T11:45:38.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 run: stack=0x557e196ad300 is done 2026-03-20T11:45:38.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196ff680:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:38.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:38.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1920a900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1920a900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1920a900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196c3030:op=0x557e1920a900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 15 stack 0x557e196c3030 end 2026-03-20T11:45:38.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 run: stack=0x557e196c3030 is done 2026-03-20T11:45:38.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196ff680:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:38.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:38.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e1970d1d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e1970d1d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196fe670:op=0x557e197017e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e1966f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e1970d1d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196fe670:op=0x557e197017e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e1966f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196c5e20:op=0x557e1970d1d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 15 stack 0x557e196c5e20 end 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 run: stack=0x557e196c5e20 is done 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196fe670:op=0x557e197017e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e1966f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196ff680:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196fe670:op=0x557e197017e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 15 stack 0x557e196fe670 end 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 run: stack=0x557e196fe670 is done 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e19686770:op=0x557e1966f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 15 stack 0x557e19686770 end 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 run: stack=0x557e19686770 is done 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196ff680:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196ff680:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196b5520:op=0x557e1970c680:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196b5520:op=0x557e1970c680:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196b5520:op=0x557e1970c680:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196b5520:op=0x557e1970c680:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 15 stack 0x557e196b5520 end 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 run: stack=0x557e196b5520 is done 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196ff680:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d3960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d3960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d3960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196be570:op=0x557e196d3960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 15 stack 0x557e196be570 end 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 run: stack=0x557e196be570 is done 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196ff680:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.033+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e192c61f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e192c61f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e192c61f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 cr:s=0x557e19701570:op=0x557e192c61f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 15 stack 0x557e19701570 end 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 run: stack=0x557e19701570 is done 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196ff680:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 cr:s=0x557e196d8aa0:op=0x557e196fff40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 cr:s=0x557e196d8aa0:op=0x557e196fff40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 cr:s=0x557e196d8aa0:op=0x557e196fff40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 cr:s=0x557e196d8aa0:op=0x557e196fff40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 15 stack 0x557e196d8aa0 end 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 run: stack=0x557e196d8aa0 is done 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196ff680:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 cr:s=0x557e196cdd00:op=0x557e196d4590:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 cr:s=0x557e196cdd00:op=0x557e196d4590:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 cr:s=0x557e196cdd00:op=0x557e196d4590:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 cr:s=0x557e196cdd00:op=0x557e196d4590:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 15 stack 0x557e196cdd00 end 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 run: stack=0x557e196cdd00 is done 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196ff680:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196d06e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196d06e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196d06e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 cr:s=0x557e196c4280:op=0x557e196d06e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 15 stack 0x557e196c4280 end 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 run: stack=0x557e196c4280 is done 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e196ff680:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 cr:s=0x557e196c7af0:op=0x557e1967b090:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 trim: trimmed bucket instance dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 15 stack 0x557e196c7af0 end 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 run: stack=0x557e196c7af0 is done 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e19670630:27BucketTrimInstanceCollectCR: operate() returned r=-22 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e1966d110:12BucketTrimCR: operate() 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 writing bucket trim marker=12:fd857d42:root::dipepu-37:head 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e196d3960:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e196d3960:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.034+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is io blocked 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.035+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e196d3960:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.035+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e196d3960:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.035+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e196d3960:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.035+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e196d3960:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.035+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e1966d110:12BucketTrimCR: operate() 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.035+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e196da040:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.035+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e196da040:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:38.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.035+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is io blocked 2026-03-20T11:45:38.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.036+0000 7f8f30ff9640 20 trim: bucket trim completed 2026-03-20T11:45:38.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.036+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e196da040:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:38.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.036+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e196da040:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:38.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.036+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e196da040:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:38.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.036+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e196da040:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:38.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.036+0000 7f8f59593900 20 cr:s=0x557e1966ced0:op=0x557e1966d110:12BucketTrimCR: operate() 2026-03-20T11:45:38.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.036+0000 7f8f59593900 4 bucket index log processing completed in 0.900767148s 2026-03-20T11:45:38.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.036+0000 7f8f59593900 15 stack 0x557e1966ced0 end 2026-03-20T11:45:38.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.036+0000 7f8f59593900 20 run: stack=0x557e1966ced0 is done 2026-03-20T11:45:38.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.041+0000 7f8f59593900 20 remove_watcher() i=3 2026-03-20T11:45:38.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.041+0000 7f8f59593900 2 removed watcher, disabling cache 2026-03-20T11:45:38.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.042+0000 7f8f59593900 20 remove_watcher() i=0 2026-03-20T11:45:38.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.042+0000 7f8f59593900 20 remove_watcher() i=6 2026-03-20T11:45:38.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.042+0000 7f8f59593900 20 remove_watcher() i=5 2026-03-20T11:45:38.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.042+0000 7f8f59593900 20 remove_watcher() i=4 2026-03-20T11:45:38.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.042+0000 7f8f59593900 20 remove_watcher() i=2 2026-03-20T11:45:38.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.042+0000 7f8f59593900 20 remove_watcher() i=7 2026-03-20T11:45:38.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.042+0000 7f8f59593900 20 remove_watcher() i=1 2026-03-20T11:45:38.046 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bilog', 'list', '--bucket', 'dipepu-58', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:38.046 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bilog', 'list', '--bucket', 'dipepu-58', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:38.046 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bilog list --bucket dipepu-58 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:45:38.122 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:38.122 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:38.215 INFO:teuthology.orchestra.run.vm02.stdout:[] 2026-03-20T11:45:38.218 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_index_log_trim ... ok 2026-03-20T11:45:38.322 INFO:rgw_multi.tests:create bucket zone=a1 name=dipepu-60 2026-03-20T11:45:38.357 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:38.357 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:38.357 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:45:38.366 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:38.367+0000 7fcfae9ab640 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2026-03-20T11:45:38.366 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:38.367+0000 7fcfbc9c7640 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2026-03-20T11:45:38.395 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:38.395 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:38.485 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:45:38.496 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774007137.857782_141.1","last_update":"2026-03-20T11:45:37.857782Z"},{"marker":"1_1774007137.912792_174.1","last_update":"2026-03-20T11:45:37.912792Z"},{"marker":"1_1774007138.336261_258.1","last_update":"2026-03-20T11:45:38.336261Z"},{"marker":"1_1774007039.336409_210.1","last_update":"2026-03-20T11:43:59.336409Z"}] 2026-03-20T11:45:38.496 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:45:38.496 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:38.496 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:38.496 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:45:38.572 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:38.572 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:38.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.647+0000 7f03d8737900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:45:38.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:38.648+0000 7f03d8737900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:45:38.656 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774007137.857782_141.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:45:37.857782Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774007137.676381_173.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:37.676381Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774007138.336261_258.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:38.336261Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774007039.336409_210.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:59.336409Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:45:38Z"} 2026-03-20T11:45:38.656 WARNING:rgw_multi.tests:zone a2 behind master: shard=1 master=1_1774007137.912792_174.1 target=1_1774007137.676381_173.1 2026-03-20T11:45:43.657 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:43.657 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:43.657 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:45:43.692 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:43.692 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:43.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:43.766+0000 7f13a0762900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:45:43.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:43.767+0000 7f13a0762900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:45:43.775 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774007137.857782_141.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:45:37.857782Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774007137.912792_174.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:37.912792Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774007138.336261_258.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:38.336261Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774007039.336409_210.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:59.336409Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:45:43Z"} 2026-03-20T11:45:43.775 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:45:43.775 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-60', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:45:43.775 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-60', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:45:43.775 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-60 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:45:43.854 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:43.854 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:43.926 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:43.928+0000 7fa6a7b30900 1 bucket sync caught up with source: 2026-03-20T11:45:43.926 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, 00000000001.30.6, , , , 00000000001.39.6, , , 00000000001.37.6, , 00000000001.116.6] 2026-03-20T11:45:43.926 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, 00000000001.30.6, , , , 00000000001.39.6, , , 00000000001.37.6, , 00000000001.116.6] 2026-03-20T11:45:43.926 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:43.928+0000 7fa6a7b30900 0 bucket checkpoint complete 2026-03-20T11:45:43.935 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-60', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:45:43.935 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-60', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:45:43.935 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-60 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:45:44.013 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:44.014 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:44.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:44.105+0000 7fad15010900 1 bucket sync caught up with source: 2026-03-20T11:45:44.103 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, 00000000001.58.6, , , , 00000000001.50.6, , , 00000000001.49.6, , 00000000001.138.6] 2026-03-20T11:45:44.103 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, 00000000001.58.6, , , , 00000000001.50.6, , , 00000000001.49.6, , 00000000001.138.6] 2026-03-20T11:45:44.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:44.105+0000 7fad15010900 0 bucket checkpoint complete 2026-03-20T11:45:44.113 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-60 zones={a2, a1} 2026-03-20T11:45:44.122 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:45:44.122 DEBUG:tasks.rgw_multi.zone_rados:o=a, v=null 2026-03-20T11:45:44.122 DEBUG:tasks.rgw_multi.zone_rados:o=b, v=null 2026-03-20T11:45:44.122 DEBUG:tasks.rgw_multi.zone_rados:o=c, v=null 2026-03-20T11:45:44.122 DEBUG:tasks.rgw_multi.zone_rados:o=d, v=null 2026-03-20T11:45:44.125 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:45:44.125 DEBUG:tasks.rgw_multi.zone_rados:o=a, v=null 2026-03-20T11:45:44.125 DEBUG:tasks.rgw_multi.zone_rados:o=b, v=null 2026-03-20T11:45:44.125 DEBUG:tasks.rgw_multi.zone_rados:o=c, v=null 2026-03-20T11:45:44.125 DEBUG:tasks.rgw_multi.zone_rados:o=d, v=null 2026-03-20T11:45:44.125 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2026-03-20T11:45:44.181 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2026-03-20T11:45:44.235 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2026-03-20T11:45:44.292 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2026-03-20T11:45:44.348 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-60 zones={a2, a1} 2026-03-20T11:45:44.348 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'reshard', '--bucket', 'dipepu-60', '--num-shards', '13', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:45:44.348 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'reshard', '--bucket', 'dipepu-60', '--num-shards', '13', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:45:44.348 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket reshard --bucket dipepu-60 --num-shards 13 --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:45:44.385 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:44.385 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:44.471 INFO:teuthology.orchestra.run.vm02.stdout:tenant: 2026-03-20T11:45:44.471 INFO:teuthology.orchestra.run.vm02.stdout:bucket name: dipepu-60 2026-03-20T11:45:44.471 INFO:teuthology.orchestra.run.vm02.stdout:start time: 2026-03-20T11:45:44.474523+0000 2026-03-20T11:45:44.473 INFO:teuthology.orchestra.run.vm02.stdout:inventory_entries: 4 2026-03-20T11:45:44.473 INFO:teuthology.orchestra.run.vm02.stdout:end time: 2026-03-20T11:45:44.477286+0000 2026-03-20T11:45:44.481 INFO:teuthology.orchestra.run.vm02.stdout:start time: 2026-03-20T11:45:44.485450+0000 2026-03-20T11:45:44.484 INFO:teuthology.orchestra.run.vm02.stdout:inc_entries: 0 2026-03-20T11:45:44.484 INFO:teuthology.orchestra.run.vm02.stdout:end time: 2026-03-20T11:45:44.488067+0000 2026-03-20T11:45:44.518 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['object', 'rm', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--bucket', 'dipepu-60', '--object', 'a', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:45:44.518 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'object', 'rm', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--bucket', 'dipepu-60', '--object', 'a', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:45:44.518 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 object rm --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --bucket dipepu-60 --object a --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:45:44.595 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:44.595 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:44.724 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['object', 'rm', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--bucket', 'dipepu-60', '--object', 'b', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:45:44.724 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'object', 'rm', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--bucket', 'dipepu-60', '--object', 'b', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:45:44.724 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 object rm --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --bucket dipepu-60 --object b --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:45:44.804 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:44.804 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:44.953 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['object', 'rm', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--bucket', 'dipepu-60', '--object', 'c', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:45:44.953 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'object', 'rm', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--bucket', 'dipepu-60', '--object', 'c', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:45:44.953 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 object rm --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --bucket dipepu-60 --object c --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:45:45.030 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:45.030 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:45.185 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['object', 'rm', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--bucket', 'dipepu-60', '--object', 'd', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:45:45.185 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'object', 'rm', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--bucket', 'dipepu-60', '--object', 'd', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:45:45.185 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 object rm --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --bucket dipepu-60 --object d --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:45:45.263 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:45.263 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:45.441 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:45.441 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:45.441 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:45:45.483 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:45.483 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:45.568 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"00000000000000000000:00000000000000008290","last_update":"2026-03-20T11:45:45.439897Z"},{"marker":"00000000000000000000:00000000000000017307","last_update":"2026-03-20T11:45:45.440994Z"},{"marker":"00000000000000000000:00000000000000012801","last_update":"2026-03-20T11:45:45.437183Z"},{"marker":"00000000000000000000:00000000000000010837","last_update":"2026-03-20T11:45:45.438983Z"}] 2026-03-20T11:45:45.569 INFO:rgw_multi.tests:starting data checkpoint for target_zone=a2 source_zone=a1 2026-03-20T11:45:45.569 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:45.569 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:45.569 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --source-zone a1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:45:45.647 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:45.648 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:45.735 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"instance_id":15702199979187085538},"markers":[{"key":0,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000007303","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:03.886627Z"}},{"key":1,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000016036","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:38.354157Z"}},{"key":2,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000011955","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:38.358744Z"}},{"key":3,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000009989","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:38.349545Z"}}]},"full_sync":{"total":0,"complete":0},"current_time":"2026-03-20T11:45:45Z"} 2026-03-20T11:45:45.735 WARNING:rgw_multi.tests:data of zone a2 behind zone a1: shard=0 master=00000000000000000000:00000000000000008290 target=00000000000000000000:00000000000000007303, shard=1 master=00000000000000000000:00000000000000017307 target=00000000000000000000:00000000000000016036, shard=2 master=00000000000000000000:00000000000000012801 target=00000000000000000000:00000000000000011955, shard=3 master=00000000000000000000:00000000000000010837 target=00000000000000000000:00000000000000009989 2026-03-20T11:45:50.736 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:50.736 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:50.736 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --source-zone a1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:45:50.775 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:50.775 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:50.876 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"instance_id":15702199979187085538},"markers":[{"key":0,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000008290","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:45.439897Z"}},{"key":1,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000017307","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:45.440994Z"}},{"key":2,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000012801","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:45.437183Z"}},{"key":3,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000010837","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:45.438983Z"}}]},"full_sync":{"total":0,"complete":0},"current_time":"2026-03-20T11:45:50Z"} 2026-03-20T11:45:50.876 INFO:rgw_multi.tests:finished data checkpoint for target_zone=a2 source_zone=a1 2026-03-20T11:45:50.876 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:50.876 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:50.876 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:45:50.953 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:50.953 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:51.057 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"00000000000000000000:00000000000000006316","last_update":"2026-03-20T11:45:45.472228Z"},{"marker":"00000000000000000000:00000000000000014485","last_update":"2026-03-20T11:45:45.930697Z"},{"marker":"00000000000000000000:00000000000000011109","last_update":"2026-03-20T11:45:45.932055Z"},{"marker":"00000000000000000000:00000000000000008718","last_update":"2026-03-20T11:45:45.933983Z"}] 2026-03-20T11:45:51.058 INFO:rgw_multi.tests:starting data checkpoint for target_zone=a1 source_zone=a2 2026-03-20T11:45:51.058 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:51.058 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:51.058 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --source-zone a2 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:45:51.137 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:51.137 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:51.230 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"instance_id":5866003269306126155},"markers":[{"key":0,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000006316","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:45.472228Z"}},{"key":1,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000014485","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:45.930697Z"}},{"key":2,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000011109","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:45.932055Z"}},{"key":3,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000008718","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:45.933983Z"}}]},"full_sync":{"total":0,"complete":0},"current_time":"2026-03-20T11:45:51Z"} 2026-03-20T11:45:51.230 INFO:rgw_multi.tests:finished data checkpoint for target_zone=a1 source_zone=a2 2026-03-20T11:45:51.230 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '50', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:45:51.230 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '50', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:45:51.230 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bilog autotrim --rgw-sync-log-trim-max-buckets 50 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 20 --debug-ms 0 2026-03-20T11:45:51.309 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:51.309 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:51.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.327+0000 7f1d4f412900 20 rados->read ofs=0 len=0 2026-03-20T11:45:51.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.328+0000 7f1d4f412900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:45:51.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.328+0000 7f1d4f412900 20 rados->read ofs=0 len=0 2026-03-20T11:45:51.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.328+0000 7f1d4f412900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:45:51.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.328+0000 7f1d4f412900 20 realm test-realm 6a789b89-c2b8-45f5-82cc-fb80234d0c6c 2026-03-20T11:45:51.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.328+0000 7f1d4f412900 20 rados->read ofs=0 len=0 2026-03-20T11:45:51.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.328+0000 7f1d4f412900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:45:51.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.328+0000 7f1d4f412900 20 rados->read ofs=0 len=0 2026-03-20T11:45:51.326 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.329+0000 7f1d4f412900 20 rados_obj.operate() r=0 bl.length=10 2026-03-20T11:45:51.326 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.329+0000 7f1d4f412900 20 rados->read ofs=0 len=0 2026-03-20T11:45:51.326 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.329+0000 7f1d4f412900 20 rados_obj.operate() r=0 bl.length=2020 2026-03-20T11:45:51.326 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.329+0000 7f1d4f412900 20 rados->read ofs=0 len=0 2026-03-20T11:45:51.326 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.330+0000 7f1d4f412900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:45:51.326 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.330+0000 7f1d4f412900 20 rados->read ofs=0 len=0 2026-03-20T11:45:51.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.330+0000 7f1d4f412900 20 rados_obj.operate() r=0 bl.length=1030 2026-03-20T11:45:51.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.330+0000 7f1d4f412900 20 zone a1 found 2026-03-20T11:45:51.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.330+0000 7f1d4f412900 4 Realm: test-realm (6a789b89-c2b8-45f5-82cc-fb80234d0c6c) 2026-03-20T11:45:51.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.330+0000 7f1d4f412900 4 ZoneGroup: a (a) 2026-03-20T11:45:51.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.330+0000 7f1d4f412900 4 Zone: a1 (4b634ebf-58f5-45bc-839a-cb4b425e773a) 2026-03-20T11:45:51.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.330+0000 7f1d4f412900 4 using period configuration: cb2fcdce-2c1b-429a-8601-a284ee3a2381:12 2026-03-20T11:45:51.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.330+0000 7f1d4f412900 20 period zonegroup name a 2026-03-20T11:45:51.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.330+0000 7f1d4f412900 20 using current period zonegroup a 2026-03-20T11:45:51.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.330+0000 7f1d4f412900 20 rados->read ofs=0 len=0 2026-03-20T11:45:51.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.330+0000 7f1d4f412900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.330+0000 7f1d4f412900 20 rados->read ofs=0 len=0 2026-03-20T11:45:51.328 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.331+0000 7f1d4f412900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.328 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.331+0000 7f1d4f412900 20 init(): pipe_map (zone=4b634ebf-58f5-45bc-839a-cb4b425e773a bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:45:51.328 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.331+0000 7f1d4f412900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.328 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.331+0000 7f1d4f412900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.328 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.331+0000 7f1d4f412900 20 rados->read ofs=0 len=0 2026-03-20T11:45:51.328 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.331+0000 7f1d4f412900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.328 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.331+0000 7f1d4f412900 20 rados->read ofs=0 len=0 2026-03-20T11:45:51.328 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.331+0000 7f1d4f412900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.328 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.331+0000 7f1d4f412900 20 init(): pipe_map (zone=51055576-6662-409d-8abb-ecaeefd3dd80 bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:45:51.328 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.331+0000 7f1d4f412900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.328 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.331+0000 7f1d4f412900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.328 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.331+0000 7f1d4f412900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.328 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.331+0000 7f1d4f412900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.328 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.331+0000 7f1d4f412900 20 started sync module instance, tier type = 2026-03-20T11:45:51.328 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.331+0000 7f1d4f412900 20 generating connection object for zone a2 id 51055576-6662-409d-8abb-ecaeefd3dd80 2026-03-20T11:45:51.328 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.331+0000 7f1d4f412900 20 started zone id=4b634ebf-58f5-45bc-839a-cb4b425e773a (name=a1) with tier type = 2026-03-20T11:45:51.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.336+0000 7f1d4f412900 20 add_watcher() i=6 2026-03-20T11:45:51.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.336+0000 7f1d4f412900 20 add_watcher() i=5 2026-03-20T11:45:51.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.336+0000 7f1d4f412900 20 add_watcher() i=0 2026-03-20T11:45:51.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.336+0000 7f1d4f412900 20 add_watcher() i=4 2026-03-20T11:45:51.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.336+0000 7f1d4f412900 20 add_watcher() i=2 2026-03-20T11:45:51.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.337+0000 7f1d4f412900 20 add_watcher() i=7 2026-03-20T11:45:51.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.338+0000 7f1d4f412900 20 add_watcher() i=3 2026-03-20T11:45:51.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.338+0000 7f1d4f412900 20 add_watcher() i=1 2026-03-20T11:45:51.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.338+0000 7f1d4f412900 2 all 8 watchers are set, enabling cache 2026-03-20T11:45:51.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.340+0000 7f1d4f412900 20 rgw_check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2026-03-20T11:45:51.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.340+0000 7f1d4f412900 20 rgw_check_secure_mon_conn(): mode 1 is insecure 2026-03-20T11:45:51.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.340+0000 7f1d4f412900 5 note: GC not initialized 2026-03-20T11:45:51.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.340+0000 7f1cf97e2640 20 reqs_thread_entry: start 2026-03-20T11:45:51.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.395+0000 7f1d4f412900 20 init_complete bucket index max shards: 11 2026-03-20T11:45:51.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.395+0000 7f1d4f412900 20 Filter name: none 2026-03-20T11:45:51.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.395+0000 7f1caaffd640 20 reqs_thread_entry: start 2026-03-20T11:45:51.392 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.395+0000 7f1ca97fa640 20 reqs_thread_entry: start 2026-03-20T11:45:51.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.397+0000 7f1d4f412900 10 rgw bucket trim manager: Watching bilog.trim 2026-03-20T11:45:51.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.397+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8dc970:12BucketTrimCR: operate() 2026-03-20T11:45:51.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.397+0000 7f1d4f412900 10 fetching active bucket counters 2026-03-20T11:45:51.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.397+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8df420:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:51.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.397+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8df420:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:51.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.397+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is io blocked 2026-03-20T11:45:51.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.397+0000 7f1d26ffd640 20 trim: get_bucket_counters: [] 2026-03-20T11:45:51.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.397+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8df420:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:51.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.397+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8df420:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:51.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.397+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8df420:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:51.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.397+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8df420:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:51.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.397+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8dc970:12BucketTrimCR: operate() 2026-03-20T11:45:51.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.397+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:51.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.397+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:51.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.397+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is io blocked 2026-03-20T11:45:51.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.398+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:51.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.398+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:51.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.398+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:51.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.398+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:51.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.398+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8dc970:12BucketTrimCR: operate() 2026-03-20T11:45:51.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.398+0000 7f1d4f412900 10 listing cold buckets from marker=12:fd857d42:root::dipepu-37:head 2026-03-20T11:45:51.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.398+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:14MetadataListCR: operate() 2026-03-20T11:45:51.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.398+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:14MetadataListCR: operate() 2026-03-20T11:45:51.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.398+0000 7f1d4f412900 20 enqueued request req=0x7f1d08002cc0 2026-03-20T11:45:51.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.398+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.398+0000 7f1d4f412900 20 req: 0x7f1d08002cc0 2026-03-20T11:45:51.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.398+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is io blocked 2026-03-20T11:45:51.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.398+0000 7f1d127fc640 20 dequeued request req=0x7f1d08002cc0 2026-03-20T11:45:51.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.398+0000 7f1d127fc640 20 RGWWQ: empty 2026-03-20T11:45:51.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.398+0000 7f1d127fc640 20 rgw async rados processor: starting metadata listing at 12:fd857d42:root::dipepu-37:head 2026-03-20T11:45:51.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.398+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-37 2026-03-20T11:45:51.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.398+0000 7f1d127fc640 20 rgw async rados processor: restarting metadata listing 2026-03-20T11:45:51.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.409+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-10 2026-03-20T11:45:51.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.409+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:45:51.413 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.416+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:45:51.419 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.422+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:45:51.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.428+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:45:51.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.433+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:45:51.436 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.439+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:45:51.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.444+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-19 2026-03-20T11:45:51.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.444+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-3 2026-03-20T11:45:51.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.444+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:45:51.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.451+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:45:51.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.456+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-17 2026-03-20T11:45:51.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.456+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:45:51.458 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.461+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-38 2026-03-20T11:45:51.458 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.461+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:45:51.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.466+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-24 2026-03-20T11:45:51.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.466+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-36 2026-03-20T11:45:51.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.466+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:45:51.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.470+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:45:51.472 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.475+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:45:51.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.479+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-44 2026-03-20T11:45:51.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.479+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-35 2026-03-20T11:45:51.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.479+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:45:51.481 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.484+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-16 2026-03-20T11:45:51.481 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.484+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:45:51.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.489+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:45:51.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.493+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 2026-03-20T11:45:51.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.497+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:45:51.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.502+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-55 2026-03-20T11:45:51.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.502+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-13 2026-03-20T11:45:51.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.502+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-48 2026-03-20T11:45:51.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.502+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-25 2026-03-20T11:45:51.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.502+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-51 2026-03-20T11:45:51.499 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.502+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:45:51.503 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.506+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:45:51.507 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.510+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-46 2026-03-20T11:45:51.507 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.510+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:45:51.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.514+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:45:51.515 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.518+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:45:51.519 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.522+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-26 2026-03-20T11:45:51.519 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.522+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-20 2026-03-20T11:45:51.519 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.522+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:45:51.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.526+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:45:51.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.530+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-1 2026-03-20T11:45:51.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.530+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:45:51.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.534+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:45:51.534 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.537+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:45:51.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.541+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-52 2026-03-20T11:45:51.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.541+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-32 2026-03-20T11:45:51.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.541+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-42 2026-03-20T11:45:51.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.541+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:45:51.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.545+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:45:51.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.548+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-7 2026-03-20T11:45:51.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.548+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-54 2026-03-20T11:45:51.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.548+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-49 2026-03-20T11:45:51.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.548+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-47 2026-03-20T11:45:51.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.548+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:45:51.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.552+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-11 2026-03-20T11:45:51.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.552+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-12 2026-03-20T11:45:51.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.552+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-41 2026-03-20T11:45:51.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.552+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-23 2026-03-20T11:45:51.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.552+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-15 2026-03-20T11:45:51.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.552+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-45 2026-03-20T11:45:51.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.552+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:45:51.552 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.555+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:45:51.555 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.558+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-9 2026-03-20T11:45:51.555 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.558+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:45:51.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.561+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-29 2026-03-20T11:45:51.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.561+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:45:51.561 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.564+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-31 2026-03-20T11:45:51.561 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.564+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:45:51.563 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.567+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-33 2026-03-20T11:45:51.563 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.567+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:45:51.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.569+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-4 2026-03-20T11:45:51.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.569+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:45:51.568 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.571+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:45:51.570 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.573+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:45:51.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.575+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-43 2026-03-20T11:45:51.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.575+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-30 2026-03-20T11:45:51.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.575+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-18 2026-03-20T11:45:51.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.575+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-2 2026-03-20T11:45:51.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.575+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:45:51.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.577+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-8 2026-03-20T11:45:51.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.577+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:45:51.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.579+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:45:51.578 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.581+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-58 2026-03-20T11:45:51.578 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.581+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:45:51.579 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.582+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:45:51.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.584+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-27 2026-03-20T11:45:51.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.584+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-28 2026-03-20T11:45:51.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.584+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-59 2026-03-20T11:45:51.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.584+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:45:51.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.585+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:45:51.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.587+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-40 2026-03-20T11:45:51.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.587+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-22 2026-03-20T11:45:51.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.587+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:45:51.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.588+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-56 2026-03-20T11:45:51.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.588+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:45:51.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d127fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:45:51.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:14MetadataListCR: operate() 2026-03-20T11:45:51.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:14MetadataListCR: operate() 2026-03-20T11:45:51.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:14MetadataListCR: operate() 2026-03-20T11:45:51.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:14MetadataListCR: operate() 2026-03-20T11:45:51.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8dc970:12BucketTrimCR: operate() 2026-03-20T11:45:51.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 4 collected 50 buckets for trim 2026-03-20T11:45:51.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 4 starting trim on bucket=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 2026-03-20T11:45:51.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 cr:s=0x55e89b919a40:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 4 starting trim on bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:45:51.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 4 starting trim on bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:45:51.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 4 starting trim on bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8e5890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 cr:s=0x55e89b919a40:op=0x55e89b8e5fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e6730:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b8e6e80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8e5890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 enqueued request req=0x55e89b8e1940 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 req: 0x55e89b8e1940 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is io blocked 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 cr:s=0x55e89b919a40:op=0x55e89b8e5fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 enqueued request req=0x55e89b8e7780 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 req: 0x55e89b8e1940 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 req: 0x55e89b8e7780 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 run: stack=0x55e89b919a40 is io blocked 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e6730:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 enqueued request req=0x55e89b8e7b20 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 req: 0x55e89b8e1940 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 req: 0x55e89b8e7780 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 req: 0x55e89b8e7b20 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is io blocked 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b8e6e80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 enqueued request req=0x55e89b8e7f40 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 req: 0x55e89b8e1940 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 req: 0x55e89b8e7780 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 req: 0x55e89b8e7b20 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 req: 0x55e89b8e7f40 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d4f412900 20 run: stack=0x55e89b8df6e0 is io blocked 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d11ffb640 20 dequeued request req=0x55e89b8e1940 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d11ffb640 20 RGWWQ: 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d11ffb640 20 req: 0x55e89b8e7780 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d11ffb640 20 req: 0x55e89b8e7b20 2026-03-20T11:45:51.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d11ffb640 20 req: 0x55e89b8e7f40 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d11ffb640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 : miss 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d11ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d117fa640 20 dequeued request req=0x55e89b8e7780 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d117fa640 20 RGWWQ: 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d117fa640 20 req: 0x55e89b8e7b20 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d117fa640 20 req: 0x55e89b8e7f40 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d117fa640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 : miss 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d117fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d10ff9640 20 dequeued request req=0x55e89b8e7b20 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d10ff9640 20 RGWWQ: 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d10ff9640 20 req: 0x55e89b8e7f40 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d07fff640 20 dequeued request req=0x55e89b8e7f40 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d07fff640 20 RGWWQ: empty 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d07fff640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 : miss 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d07fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d10ff9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 : miss 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.589+0000 7f1d10ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d11ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=437 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d11ffb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 info.flags=0x17 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d11ffb640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 to cache LRU end 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d11ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d11ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d11ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d11ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-60 : miss 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d11ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d117fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d117fa640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 info.flags=0x17 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d117fa640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 to cache LRU end 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d117fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d117fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d117fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d117fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-55 : miss 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d117fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d07fff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d07fff640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 info.flags=0x17 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d07fff640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 to cache LRU end 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d07fff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d07fff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d07fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d07fff640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-31 : miss 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d07fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d10ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d10ff9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 info.flags=0x17 2026-03-20T11:45:51.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d10ff9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 to cache LRU end 2026-03-20T11:45:51.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d10ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d10ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:45:51.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d10ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:45:51.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d10ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-28 : miss 2026-03-20T11:45:51.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d10ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d10ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d117fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d10ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-28 info.flags=0x0 2026-03-20T11:45:51.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d10ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-28 to cache LRU end 2026-03-20T11:45:51.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d10ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-28 : miss 2026-03-20T11:45:51.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d10ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d117fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-55 info.flags=0x0 2026-03-20T11:45:51.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d117fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-55 to cache LRU end 2026-03-20T11:45:51.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d117fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-55 : miss 2026-03-20T11:45:51.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d117fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d11ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d11ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-60 info.flags=0x0 2026-03-20T11:45:51.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d11ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-60 to cache LRU end 2026-03-20T11:45:51.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d11ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-60 : miss 2026-03-20T11:45:51.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d11ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d07fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.590+0000 7f1d07fff640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-31 info.flags=0x0 2026-03-20T11:45:51.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d07fff640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-31 to cache LRU end 2026-03-20T11:45:51.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d07fff640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-31 : miss 2026-03-20T11:45:51.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d07fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d10ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d10ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-28 info.flags=0x0 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d10ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-28 to cache LRU end 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d10ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding source pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d10ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding dest pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d10ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d10ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d10ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d117fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e6730:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e6730:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e6730:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e6730:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d117fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-55 info.flags=0x0 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b8e6be0:op=0x55e89b532a90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d117fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-55 to cache LRU end 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d117fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding source pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b8e6be0:op=0x55e89b5330b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b8e6be0:op=0x55e89b5330b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d117fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding dest pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d117fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d117fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d117fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d07fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d07fff640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-31 info.flags=0x0 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d07fff640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-31 to cache LRU end 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d07fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding source pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d07fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding dest pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d07fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d07fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d07fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d11ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 46ae6122e8bd553d726985e9f291a316fcac7e23d55927dbc8d13a7d8ed93f85 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d11ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-60 info.flags=0x0 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d11ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-60 to cache LRU end 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:46ae6122e8bd553d726985e9f291a316fcac7e23d55927dbc8d13a7d8ed93f85 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d11ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20): adding source pipe: {s={b=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d11ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20): adding dest pipe: {s={b=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 57f734d16504f043f7d8960f9e03d62d13e09f1dd2cb6678b90439e1ac80f6bc 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=57f734d16504f043f7d8960f9e03d62d13e09f1dd2cb6678b90439e1ac80f6bc 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d11ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d11ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=0, curl_handle=0x55e89b534cc0 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 run: stack=0x55e89b8e6be0 is io blocked 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b919a40:op=0x55e89b8e5fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b8e6e80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b919a40:op=0x55e89b8e5fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b8e6e80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b919a40:op=0x55e89b8e5fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b8e6e80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b919a40:op=0x55e89b8e5fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b8e6e80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.596 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b919a40:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d11ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b8e7330:op=0x55e89b8e5fe0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b919a40:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 run: stack=0x55e89b919a40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b479f80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 run: stack=0x55e89b8df6e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b8e7330:op=0x55e89b47a5a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8e5890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b47abd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b8e7330:op=0x55e89b47a5a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 3b89f1ae17eda80c9b7e3dae28acd988082d7598145ef6fa0e9356e6910df2d8 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:3b89f1ae17eda80c9b7e3dae28acd988082d7598145ef6fa0e9356e6910df2d8 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = cb684effe8bedd123c30711359362578deb140015bc63475774f7f69b309ae4d 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=cb684effe8bedd123c30711359362578deb140015bc63475774f7f69b309ae4d 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1ca97fa640 20 link_request req_data=0x55e89b5356e0 req_data->id=0, curl_handle=0x55e89b534cc0 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=1, curl_handle=0x55e89b47c780 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 run: stack=0x55e89b8e7330 is io blocked 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8e5890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b47abd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.597 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = de69947fa4b0a11701bde6a7ff02c12bd767d774d9fec1ee9d4c51424fdf254b 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:de69947fa4b0a11701bde6a7ff02c12bd767d774d9fec1ee9d4c51424fdf254b 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = e92a4302de323af15adeecf6dc7f4d460e357d77e058c09746a99c33fae9c33f 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=e92a4302de323af15adeecf6dc7f4d460e357d77e058c09746a99c33fae9c33f 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=2, curl_handle=0x55e89b47fb60 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 run: stack=0x55e89b8e7090 is io blocked 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8e5890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8e5890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8e5890:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8e8360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8e8360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.598 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 23814d881227d801a4794b745a5a46b5b5dd52208caa14ef757684c20ed3e730 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:23814d881227d801a4794b745a5a46b5b5dd52208caa14ef757684c20ed3e730 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 4e0d200a68584f1166885c854aecd122954e5ae296dc6624474d382337da6836 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=4e0d200a68584f1166885c854aecd122954e5ae296dc6624474d382337da6836 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=3, curl_handle=0x55e89b8e9f10 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is io blocked 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1ca97fa640 20 link_request req_data=0x55e89b47ced0 req_data->id=1, curl_handle=0x55e89b47c780 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1ca97fa640 20 link_request req_data=0x55e89b4802b0 req_data->id=2, curl_handle=0x55e89b47fb60 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.591+0000 7f1ca97fa640 20 link_request req_data=0x55e89b8ea620 req_data->id=3, curl_handle=0x55e89b8e9f10 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.593+0000 7f1d4f412900 20 cr:s=0x55e89b8e6be0:op=0x55e89b5330b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.593+0000 7f1d4f412900 20 cr:s=0x55e89b8e6be0:op=0x55e89b5330b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.593+0000 7f1d4f412900 20 cr:s=0x55e89b8e6be0:op=0x55e89b532a90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.593+0000 7f1d4f412900 15 stack 0x55e89b8e6be0 end 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.593+0000 7f1d4f412900 20 run: stack=0x55e89b8e6be0 is done 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.593+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.593+0000 7f1d4f412900 20 collect(): s=0x55e89b7ed970 stack=0x55e89b8e6be0 is complete 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.593+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.593+0000 7f1d4f412900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.593+0000 7f1d4f412900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.593+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.593+0000 7f1d4f412900 15 stack 0x55e89b7ed970 end 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.593+0000 7f1d4f412900 20 stack->operate() returned ret=-22 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.593+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is done 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.593+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.593+0000 7f1d4f412900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.593+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.593+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.593+0000 7f1d4f412900 4 starting trim on bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.593+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b532a90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.593+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b532a90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.593+0000 7f1d4f412900 20 enqueued request req=0x55e89b8e7b20 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.593+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.593+0000 7f1d4f412900 20 req: 0x55e89b8e7b20 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.593+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is io blocked 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.593+0000 7f1d077fe640 20 dequeued request req=0x55e89b8e7b20 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.593+0000 7f1d077fe640 20 RGWWQ: empty 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.593+0000 7f1d077fe640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 : miss 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d077fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b47abd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b47abd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b479f80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d4f412900 15 stack 0x55e89b8e7090 end 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d4f412900 20 run: stack=0x55e89b8e7090 is done 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d4f412900 20 collect(): s=0x55e89b8df6e0 stack=0x55e89b8e7090 is complete 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d4f412900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d4f412900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:51.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d4f412900 15 stack 0x55e89b8df6e0 end 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d4f412900 20 stack->operate() returned ret=-22 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d4f412900 20 run: stack=0x55e89b8df6e0 is done 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d4f412900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d4f412900 4 starting trim on bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b47e6b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b47e6b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d4f412900 20 enqueued request req=0x55e89b8e7780 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d4f412900 20 req: 0x55e89b8e7780 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d4f412900 20 run: stack=0x55e89b8df6e0 is io blocked 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d06ffd640 20 dequeued request req=0x55e89b8e7780 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d06ffd640 20 RGWWQ: empty 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d06ffd640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 : miss 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d06ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d077fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d077fe640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 info.flags=0x17 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d077fe640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 to cache LRU end 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d077fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d077fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d077fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d077fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-39 : miss 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d077fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.594+0000 7f1d077fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d077fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-39 info.flags=0x0 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d077fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-39 to cache LRU end 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d077fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-39 : miss 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d077fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d06ffd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d06ffd640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 info.flags=0x17 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d06ffd640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 to cache LRU end 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d06ffd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d06ffd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d06ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d06ffd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-8 : miss 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d06ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d06ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d06ffd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-8 info.flags=0x0 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d06ffd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-8 to cache LRU end 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d06ffd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-8 : miss 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d06ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d077fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d077fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-39 info.flags=0x0 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d077fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-39 to cache LRU end 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d077fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding source pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d077fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding dest pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d077fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d077fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d077fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b532a90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b532a90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b532a90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b532a90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b47abd0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b532a90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b532a90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = fbcb01a5bd1b59aed7ece4f8ef90daaf3d2f69ad7cf5d7da6a0ec4153e5e1eb3 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:fbcb01a5bd1b59aed7ece4f8ef90daaf3d2f69ad7cf5d7da6a0ec4153e5e1eb3 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = ce8206d2affda29b9b2927a2aef8471d4292254aa5625ff87ca7f0ad94650c3a 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=ce8206d2affda29b9b2927a2aef8471d4292254aa5625ff87ca7f0ad94650c3a 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=4, curl_handle=0x55e89b47fb60 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 run: stack=0x55e89b8e7090 is io blocked 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d06ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d06ffd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-8 info.flags=0x0 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d06ffd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-8 to cache LRU end 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d06ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding source pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d06ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding dest pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d06ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d06ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d06ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b47e6b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b47e6b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b47e6b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b47e6b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 cr:s=0x55e89b8e6be0:op=0x55e89b479f80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 run: stack=0x55e89b8df6e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 cr:s=0x55e89b8e6be0:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 cr:s=0x55e89b8e6be0:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = e3f3446612734ce8dd355ebb56941726123c2c5724a3708de152f7106f6170e8 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:e3f3446612734ce8dd355ebb56941726123c2c5724a3708de152f7106f6170e8 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = b36a235feeea44893325642dd54a2ce582398bb9b0e7f990dad1e3e30f291189 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b36a235feeea44893325642dd54a2ce582398bb9b0e7f990dad1e3e30f291189 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=5, curl_handle=0x55e89b534cc0 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1d4f412900 20 run: stack=0x55e89b8e6be0 is io blocked 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.595+0000 7f1ca97fa640 20 link_request req_data=0x55e89b4802b0 req_data->id=4, curl_handle=0x55e89b47fb60 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1ca97fa640 20 link_request req_data=0x55e89b5356e0 req_data->id=5, curl_handle=0x55e89b534cc0 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b8e7330:op=0x55e89b47a5a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b8e7330:op=0x55e89b47a5a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8e8360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b8e7330:op=0x55e89b47a5a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8e8360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b8e7330:op=0x55e89b47a5a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8e8360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b8e7330:op=0x55e89b8e5fe0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 15 stack 0x55e89b8e7330 end 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 run: stack=0x55e89b8e7330 is done 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8e8360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b919a40:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 collect(): s=0x55e89b919a40 stack=0x55e89b8e7330 is complete 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 10 trimming bilogs for bucket=:dipepu-55[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15]) markers=[,,,,,00000000006.98.6,,,,,], shards=11 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8e5890:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 15 stack 0x55e89b482230 end 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is done 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b919a40:op=0x55e89b946220:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 10 trimming bilog shard 5 of :dipepu-55[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15]) at marker 00000000006.98.6 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 run: stack=0x55e89b919a40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 collect(): s=0x55e89b8e14b0 stack=0x55e89b482230 is complete 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 10 min_generation is 1 2026-03-20T11:45:51.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b8e7330:op=0x55e89b941eb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8e5890:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.0 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.2 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.3 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.4 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.5 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.6 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.7 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.8 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.9 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.10 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b8e7330:op=0x55e89b941eb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15.5 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 run: stack=0x55e89b8e7330 is io blocked 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8e5fe0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b942d20:op=0x55e89b47a5a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b94c560:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b8edd30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b943670:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b94f8f0:op=0x55e89b94be70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b94fa30:op=0x55e89b9480d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b94ea10:op=0x55e89b94eeb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b944070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b94cdd0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b9474d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8e5fe0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is io blocked 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b942d20:op=0x55e89b47a5a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 run: stack=0x55e89b942d20 is io blocked 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b94c560:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 run: stack=0x55e89b943030 is io blocked 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b8edd30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 run: stack=0x55e89b47ccf0 is io blocked 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b943670:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 run: stack=0x55e89b8ea3f0 is io blocked 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b94f8f0:op=0x55e89b94be70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 run: stack=0x55e89b94f8f0 is io blocked 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b94fa30:op=0x55e89b9480d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 run: stack=0x55e89b94fa30 is io blocked 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b94ea10:op=0x55e89b94eeb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 run: stack=0x55e89b94ea10 is io blocked 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b944070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 run: stack=0x55e89b947330 is io blocked 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b94cdd0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 run: stack=0x55e89b8f22f0 is io blocked 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b9474d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.596+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is io blocked 2026-03-20T11:45:51.604 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:51.592+0000 7fd05ab03640 -1 req 14409937438986822247 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:51.604 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:51.592+0000 7fd0a8b9f640 -1 req 17517327694051583810 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.598+0000 7f1d4f412900 20 cr:s=0x55e89b942d20:op=0x55e89b47a5a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.598+0000 7f1d4f412900 20 cr:s=0x55e89b942d20:op=0x55e89b47a5a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.598+0000 7f1d4f412900 20 cr:s=0x55e89b942d20:op=0x55e89b47a5a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.598+0000 7f1d4f412900 20 cr:s=0x55e89b942d20:op=0x55e89b47a5a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.598+0000 7f1d4f412900 15 stack 0x55e89b942d20 end 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.598+0000 7f1d4f412900 20 run: stack=0x55e89b942d20 is done 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.598+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8e5890:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.598+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b8edd30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b8edd30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b8edd30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b8edd30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 15 stack 0x55e89b47ccf0 end 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 run: stack=0x55e89b47ccf0 is done 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8e5890:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b94c560:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b94c560:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b94c560:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b94c560:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 15 stack 0x55e89b943030 end 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 run: stack=0x55e89b943030 is done 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8e5890:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b94cdd0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b94cdd0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 cr:s=0x55e89b94f8f0:op=0x55e89b94be70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b944070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b94cdd0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 cr:s=0x55e89b94f8f0:op=0x55e89b94be70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b944070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b94cdd0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 15 stack 0x55e89b8f22f0 end 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 run: stack=0x55e89b8f22f0 is done 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 cr:s=0x55e89b94f8f0:op=0x55e89b94be70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b944070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8e5890:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 cr:s=0x55e89b94f8f0:op=0x55e89b94be70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 15 stack 0x55e89b94f8f0 end 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 run: stack=0x55e89b94f8f0 is done 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b944070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 15 stack 0x55e89b947330 end 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 run: stack=0x55e89b947330 is done 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8e5890:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8e5890:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.599+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 20 cr:s=0x55e89b94fa30:op=0x55e89b9480d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 20 cr:s=0x55e89b94fa30:op=0x55e89b9480d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 20 cr:s=0x55e89b94fa30:op=0x55e89b9480d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 20 cr:s=0x55e89b94fa30:op=0x55e89b9480d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 15 stack 0x55e89b94fa30 end 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 20 run: stack=0x55e89b94fa30 is done 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8e5890:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b943670:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b943670:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b943670:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b943670:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 15 stack 0x55e89b8ea3f0 end 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 20 run: stack=0x55e89b8ea3f0 is done 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8e5890:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 20 cr:s=0x55e89b94ea10:op=0x55e89b94eeb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 20 cr:s=0x55e89b94ea10:op=0x55e89b94eeb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 20 cr:s=0x55e89b94ea10:op=0x55e89b94eeb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 20 cr:s=0x55e89b94ea10:op=0x55e89b94eeb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 15 stack 0x55e89b94ea10 end 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 20 run: stack=0x55e89b94ea10 is done 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8e5890:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8e5fe0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8e5fe0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8e5fe0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8e5fe0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 15 stack 0x55e89b482230 end 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is done 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8e5890:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.600+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 cr:s=0x55e89b8e7330:op=0x55e89b941eb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 cr:s=0x55e89b8e7330:op=0x55e89b941eb0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 15 stack 0x55e89b8e7330 end 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 run: stack=0x55e89b8e7330 is done 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 cr:s=0x55e89b919a40:op=0x55e89b946220:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b9474d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 cr:s=0x55e89b919a40:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 trim: trimmed bucket instance dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 15 stack 0x55e89b919a40 end 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 run: stack=0x55e89b919a40 is done 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b9474d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b9474d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 4 starting trim on bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b9474d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 15 stack 0x55e89b946f40 end 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is done 2026-03-20T11:45:51.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b941eb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8e5890:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b941eb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 enqueued request req=0x55e89b8e1710 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 req: 0x55e89b8e1710 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 run: stack=0x55e89b8ea3f0 is io blocked 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8e8360:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8e8360:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 enqueued request req=0x55e89b8f2d60 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 req: 0x55e89b8e1710 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 req: 0x55e89b8f2d60 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is io blocked 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d067fc640 20 dequeued request req=0x55e89b8e1710 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d067fc640 20 RGWWQ: 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d067fc640 20 req: 0x55e89b8f2d60 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d067fc640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 : miss 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d067fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d05ffb640 20 dequeued request req=0x55e89b8f2d60 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d05ffb640 20 RGWWQ: empty 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d067fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d067fc640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 info.flags=0x17 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d067fc640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 to cache LRU end 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d067fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d067fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d067fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d067fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-16 : miss 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.601+0000 7f1d067fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d067fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d067fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-16 info.flags=0x0 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d067fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-16 to cache LRU end 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d067fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-16 : miss 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d067fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d067fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d067fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-16 info.flags=0x0 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d067fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-16 to cache LRU end 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d067fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding source pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d067fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding dest pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d067fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d067fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d067fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b941eb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b941eb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b941eb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b941eb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b8e5890:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 20 run: stack=0x55e89b8ea3f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b47a5a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b47a5a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.607 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d05ffb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 info.flags=0x11 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d05ffb640 10 rgw async rados processor: moving a1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 to cache LRU end 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d05ffb640 10 rgw async rados processor: distributing notification oid=a1.rgw.control:notify.6 cni=[op: 0, obj: a1.rgw.meta:root:.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20, ofs0, ns] 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 469ce04d135aef770e2436b0e28823cae58bd69e9aaffd002cf94a1486d3ac29 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:469ce04d135aef770e2436b0e28823cae58bd69e9aaffd002cf94a1486d3ac29 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 7cd4a45cc9736cfff0f26e1835000389a500fd37c771e3f6512d7ce01f660427 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=7cd4a45cc9736cfff0f26e1835000389a500fd37c771e3f6512d7ce01f660427 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=6, curl_handle=0x55e89b8e9f10 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is io blocked 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.602+0000 7f1ca97fa640 20 link_request req_data=0x55e89b8ea620 req_data->id=6, curl_handle=0x55e89b8e9f10 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.603+0000 7f1d267fc640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 343597383927 cookie 94457529582432 notifier 6341 bl.length()=617 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.603+0000 7f1d267fc640 10 rgw watcher librados: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 info.flags=0x11 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.603+0000 7f1d267fc640 10 rgw watcher librados: moving a1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 to cache LRU end 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.605+0000 7f1d05ffb640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-60[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20]): orig_sources= new_sources= 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.605+0000 7f1d05ffb640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-60[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20]): potential sources added=[] removed=[] 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.605+0000 7f1d05ffb640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-60[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20]): orig_dests= new_dests= 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.605+0000 7f1d05ffb640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-60[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20]): potential dests added=[] removed=[] 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.605+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8e8360:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.605+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8e8360:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.605+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8e8360:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.605+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8e8360:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.605+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.605+0000 7f1d4f412900 20 trim: trimmed bucket instance dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.605+0000 7f1d4f412900 15 stack 0x55e89b8e14b0 end 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.605+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is done 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.605+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.605+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.605+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.605+0000 7f1d4f412900 4 starting trim on bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.605+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b944070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.605+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b944070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.605+0000 7f1d4f412900 20 enqueued request req=0x55e89b946d50 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.605+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.605+0000 7f1d4f412900 20 req: 0x55e89b946d50 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.605+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is io blocked 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.605+0000 7f1cfafe5640 20 dequeued request req=0x55e89b946d50 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.605+0000 7f1cfafe5640 20 RGWWQ: empty 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.605+0000 7f1cfafe5640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 : miss 2026-03-20T11:45:51.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.605+0000 7f1cfafe5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1cfafe5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=314 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1cfafe5640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 info.flags=0x17 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1cfafe5640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 to cache LRU end 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1cfafe5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1cfafe5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1cfafe5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1cfafe5640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-43 : miss 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1cfafe5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1cfafe5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1cfafe5640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-43 info.flags=0x0 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1cfafe5640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-43 to cache LRU end 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1cfafe5640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-43 : miss 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1cfafe5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1cfafe5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1cfafe5640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-43 info.flags=0x0 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1cfafe5640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-43 to cache LRU end 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1cfafe5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding source pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1cfafe5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding dest pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1cfafe5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1cfafe5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1cfafe5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b944070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b944070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b944070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b944070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b943670:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b94edf0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b94edf0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = d3dbfe62163d54cacebe7b5bf804c3567e01b41ccaab7e00ab2f68740693e413 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:d3dbfe62163d54cacebe7b5bf804c3567e01b41ccaab7e00ab2f68740693e413 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 3b90638296484c231eb9414b99dce088373ebdfdb4285b969f551b9be70ab4be 2026-03-20T11:45:51.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=3b90638296484c231eb9414b99dce088373ebdfdb4285b969f551b9be70ab4be 2026-03-20T11:45:51.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=7, curl_handle=0x55e89b47c780 2026-03-20T11:45:51.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1d4f412900 20 run: stack=0x55e89b947330 is io blocked 2026-03-20T11:45:51.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.606+0000 7f1ca97fa640 20 link_request req_data=0x7f1cf0017530 req_data->id=7, curl_handle=0x55e89b47c780 2026-03-20T11:45:51.610 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:51.606+0000 7fd09a382640 -1 req 5791059487105178432 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:51.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b532a90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b532a90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b8e6be0:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b532a90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b8e6be0:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b532a90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b8e6be0:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b47abd0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 15 stack 0x55e89b8e7090 end 2026-03-20T11:45:51.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 run: stack=0x55e89b8e7090 is done 2026-03-20T11:45:51.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b8e6be0:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 collect(): s=0x55e89b7ed970 stack=0x55e89b8e7090 is complete 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 10 trimming bilogs for bucket=:dipepu-39[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44]) markers=[,,,,,,,,00000000001.10.6,,], shards=11 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b8e6be0:op=0x55e89b479f80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 15 stack 0x55e89b8e6be0 end 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 run: stack=0x55e89b8e6be0 is done 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b9461a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 10 trimming bilog shard 8 of :dipepu-39[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44]) at marker 00000000001.10.6 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 collect(): s=0x55e89b8df6e0 stack=0x55e89b8e6be0 is complete 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 10 trimming bilogs for bucket=:dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) markers=[,00000000002.12.6,,00000000001.5.6,,00000000001.4.6,00000000002.12.6,00000000001.8.6,00000000001.14.6,00000000001.9.6,], shards=11 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b8eec30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b47abd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 10 trimming bilog shard 1 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000002.12.6 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 10 trimming bilog shard 3 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.5.6 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 10 trimming bilog shard 5 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.4.6 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 10 trimming bilog shard 6 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000002.12.6 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 10 trimming bilog shard 7 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.8.6 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 10 trimming bilog shard 8 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.14.6 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 10 trimming bilog shard 9 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.9.6 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 run: stack=0x55e89b8df6e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b8eec30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44.8 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 run: stack=0x55e89b8e7090 is io blocked 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b8e6be0:op=0x55e89b952bc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b94f8f0:op=0x55e89b8ebda0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b9521a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b94dbe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b951480:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b942d20:op=0x55e89b945050:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.638 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8e8360:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b8e6be0:op=0x55e89b952bc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.1 2026-03-20T11:45:51.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 run: stack=0x55e89b8e6be0 is io blocked 2026-03-20T11:45:51.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b94f8f0:op=0x55e89b8ebda0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.3 2026-03-20T11:45:51.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 run: stack=0x55e89b94f8f0 is io blocked 2026-03-20T11:45:51.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b9521a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.5 2026-03-20T11:45:51.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 run: stack=0x55e89b8f22f0 is io blocked 2026-03-20T11:45:51.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b94dbe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.6 2026-03-20T11:45:51.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 run: stack=0x55e89b943030 is io blocked 2026-03-20T11:45:51.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b951480:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.7 2026-03-20T11:45:51.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 run: stack=0x55e89b47ccf0 is io blocked 2026-03-20T11:45:51.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b942d20:op=0x55e89b945050:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.8 2026-03-20T11:45:51.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 run: stack=0x55e89b942d20 is io blocked 2026-03-20T11:45:51.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8e8360:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.9 2026-03-20T11:45:51.639 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.640+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is io blocked 2026-03-20T11:45:51.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 cr:s=0x55e89b8e6be0:op=0x55e89b952bc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 cr:s=0x55e89b8e6be0:op=0x55e89b952bc0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 15 stack 0x55e89b8e6be0 end 2026-03-20T11:45:51.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 run: stack=0x55e89b8e6be0 is done 2026-03-20T11:45:51.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b47abd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 run: stack=0x55e89b8df6e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b951480:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b951480:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 15 stack 0x55e89b47ccf0 end 2026-03-20T11:45:51.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 run: stack=0x55e89b47ccf0 is done 2026-03-20T11:45:51.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b47abd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 run: stack=0x55e89b8df6e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b8eec30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b8eec30:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 15 stack 0x55e89b8e7090 end 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 run: stack=0x55e89b8e7090 is done 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b9461a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 trim: trimmed bucket instance dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 15 stack 0x55e89b7ed970 end 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is done 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 4 starting trim on bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8ef510:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8ef510:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 enqueued request req=0x55e89b8e7f40 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 req: 0x55e89b8e7f40 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is io blocked 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d057fa640 20 dequeued request req=0x55e89b8e7f40 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d057fa640 20 RGWWQ: empty 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d057fa640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 : miss 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d057fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8e8360:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8e8360:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.642+0000 7f1d4f412900 15 stack 0x55e89b482230 end 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is done 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b47abd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 run: stack=0x55e89b8df6e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b94dbe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b94dbe0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 15 stack 0x55e89b943030 end 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 run: stack=0x55e89b943030 is done 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b47abd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 run: stack=0x55e89b8df6e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d057fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d057fa640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 info.flags=0x17 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d057fa640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 to cache LRU end 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d057fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d057fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d057fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d057fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-10 : miss 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d057fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b9521a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b9521a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 15 stack 0x55e89b8f22f0 end 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 run: stack=0x55e89b8f22f0 is done 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b47abd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 run: stack=0x55e89b8df6e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 cr:s=0x55e89b94f8f0:op=0x55e89b8ebda0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 cr:s=0x55e89b94f8f0:op=0x55e89b8ebda0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 15 stack 0x55e89b94f8f0 end 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 run: stack=0x55e89b94f8f0 is done 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 cr:s=0x55e89b942d20:op=0x55e89b945050:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 cr:s=0x55e89b942d20:op=0x55e89b945050:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 15 stack 0x55e89b942d20 end 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 run: stack=0x55e89b942d20 is done 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b47abd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b47abd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 cr:s=0x55e89b8df6e0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 trim: trimmed bucket instance dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 15 stack 0x55e89b8df6e0 end 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 run: stack=0x55e89b8df6e0 is done 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 4 starting trim on bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b9480d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b9480d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 enqueued request req=0x55e89b946940 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 req: 0x55e89b946940 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d4f412900 20 run: stack=0x55e89b8f22f0 is io blocked 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d04ff9640 20 dequeued request req=0x55e89b946940 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d04ff9640 20 RGWWQ: empty 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d04ff9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 : miss 2026-03-20T11:45:51.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d04ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d057fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d057fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-10 info.flags=0x0 2026-03-20T11:45:51.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d057fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-10 to cache LRU end 2026-03-20T11:45:51.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d057fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-10 : miss 2026-03-20T11:45:51.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d057fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d04ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:45:51.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d04ff9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 info.flags=0x17 2026-03-20T11:45:51.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d04ff9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 to cache LRU end 2026-03-20T11:45:51.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d04ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d04ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d04ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:45:51.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d04ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-3 : miss 2026-03-20T11:45:51.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d04ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.643 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:51.644+0000 7fd0a0b8f640 -1 req 13788530067797259174 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d057fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d057fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-10 info.flags=0x0 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d057fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-10 to cache LRU end 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d057fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding source pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d057fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding dest pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d057fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d057fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.643+0000 7f1d057fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8ef510:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8ef510:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8ef510:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8ef510:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b9461a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b47abd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b47abd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 6bfe81aef260849b1d4366a53661afc4429a2e7d6f8c3756b7c2ec5b4f29ef2f 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:6bfe81aef260849b1d4366a53661afc4429a2e7d6f8c3756b7c2ec5b4f29ef2f 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.644 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 826805b8d52020d4784db9a6dcae8febf459ad69d6e0c142853b1c6e86313775 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=826805b8d52020d4784db9a6dcae8febf459ad69d6e0c142853b1c6e86313775 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=8, curl_handle=0x55e89b534cc0 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 run: stack=0x55e89b943030 is io blocked 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d04ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d04ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-3 info.flags=0x0 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d04ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-3 to cache LRU end 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d04ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-3 : miss 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d04ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1ca97fa640 20 link_request req_data=0x55e89b5356e0 req_data->id=8, curl_handle=0x55e89b534cc0 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d04ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d04ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-3 info.flags=0x0 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d04ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-3 to cache LRU end 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d04ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding source pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d04ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding dest pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d04ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d04ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d04ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b9480d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b9480d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b9480d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b9480d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b479f30:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 run: stack=0x55e89b8f22f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8e5f60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8e5f60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 1d039153c1e1d427e12cd18324e93581dd815396c29e8bffe2b956d6e8680ded 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.645 INFO:teuthology.orchestra.run.vm02.stderr:1d039153c1e1d427e12cd18324e93581dd815396c29e8bffe2b956d6e8680ded 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 01d15bc17bdf6a940248ef35715a2ec5b7fc63652a4082256ca0e2e5967b56b6 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=01d15bc17bdf6a940248ef35715a2ec5b7fc63652a4082256ca0e2e5967b56b6 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=9, curl_handle=0x55e89b47fb60 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is io blocked 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.644+0000 7f1ca97fa640 20 link_request req_data=0x55e89b4802b0 req_data->id=9, curl_handle=0x55e89b47fb60 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.646+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b47a5a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.646+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b47a5a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.646+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b47a5a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.646+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b47a5a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.646+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b8e5890:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.646+0000 7f1d4f412900 15 stack 0x55e89b946f40 end 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.646+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is done 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.646+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.646+0000 7f1d4f412900 20 collect(): s=0x55e89b8ea3f0 stack=0x55e89b946f40 is complete 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.646+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.646+0000 7f1d4f412900 10 trimming bilogs for bucket=:dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) markers=[,,,,,00000000003.17.10,,,00000000031.93.9,,], shards=11 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.646+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8f0ec0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.646+0000 7f1d4f412900 10 trimming bilog shard 5 of :dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) at marker 00000000003.17.10 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.646+0000 7f1d4f412900 10 trimming bilog shard 8 of :dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) at marker 00000000031.93.9 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.646+0000 7f1d4f412900 20 run: stack=0x55e89b8ea3f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.646+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b8eec30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.646+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b8ef460:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.646+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b8eec30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.646+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21.5 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.646+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is io blocked 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.646+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b8ef460:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.646+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21.8 2026-03-20T11:45:51.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.646+0000 7f1d4f412900 20 run: stack=0x55e89b8e7090 is io blocked 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b8eec30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b8eec30:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d4f412900 15 stack 0x55e89b946f40 end 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is done 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8f0ec0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d4f412900 20 run: stack=0x55e89b8ea3f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b8ef460:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b8ef460:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d4f412900 15 stack 0x55e89b8e7090 end 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d4f412900 20 run: stack=0x55e89b8e7090 is done 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8f0ec0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d4f412900 20 trim: trimmed bucket instance dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d4f412900 15 stack 0x55e89b8ea3f0 end 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d4f412900 20 run: stack=0x55e89b8ea3f0 is done 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d4f412900 4 starting trim on bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8eec30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8eec30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d4f412900 20 enqueued request req=0x55e89b47c240 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d4f412900 20 req: 0x55e89b47c240 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d4f412900 20 run: stack=0x55e89b8ea3f0 is io blocked 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d047f8640 20 dequeued request req=0x55e89b47c240 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d047f8640 20 RGWWQ: empty 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d047f8640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 : miss 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.647+0000 7f1d047f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b94edf0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b94edf0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b943670:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 15 stack 0x55e89b947330 end 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 20 run: stack=0x55e89b947330 is done 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 20 collect(): s=0x55e89b8e14b0 stack=0x55e89b947330 is complete 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 15 stack 0x55e89b8e14b0 end 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 20 stack->operate() returned ret=-22 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is done 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 4 starting trim on bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8ef460:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8ef460:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 20 enqueued request req=0x55e89b946d50 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 20 req: 0x55e89b946d50 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is io blocked 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d03ff7640 20 dequeued request req=0x55e89b946d50 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d03ff7640 20 RGWWQ: empty 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d03ff7640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 : miss 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d03ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d047f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d047f8640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 info.flags=0x17 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d047f8640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 to cache LRU end 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d047f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d047f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d047f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d047f8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-9 : miss 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d047f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d03ff7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d03ff7640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 info.flags=0x17 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d03ff7640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 to cache LRU end 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d03ff7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d03ff7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d03ff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d03ff7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-38 : miss 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d03ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d047f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d047f8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-9 info.flags=0x0 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d047f8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-9 to cache LRU end 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d047f8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-9 : miss 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d047f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d03ff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d03ff7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-38 info.flags=0x0 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d03ff7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-38 to cache LRU end 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d03ff7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-38 : miss 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d03ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d047f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d047f8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-9 info.flags=0x0 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d047f8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-9 to cache LRU end 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d047f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding source pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d047f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding dest pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d047f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d047f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d047f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8eec30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8eec30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8eec30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8eec30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b8e5890:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 20 run: stack=0x55e89b8ea3f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b47a5a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.648+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b47a5a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = d2ba8a1a659c6130d9247da1f1b611b5832f218da7c2153e37968fa246c11bc3 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:d2ba8a1a659c6130d9247da1f1b611b5832f218da7c2153e37968fa246c11bc3 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 7eee2667fcaf32c79d4f8e24964e814c86a2b3b461d02ce93600e9ac8e0bbb7e 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=7eee2667fcaf32c79d4f8e24964e814c86a2b3b461d02ce93600e9ac8e0bbb7e 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=10, curl_handle=0x55e89b47c780 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 20 run: stack=0x55e89b947330 is io blocked 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d03ff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d03ff7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-38 info.flags=0x0 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d03ff7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-38 to cache LRU end 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d03ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding source pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d03ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding dest pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1ca97fa640 20 link_request req_data=0x7f1cf0017530 req_data->id=10, curl_handle=0x55e89b47c780 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d03ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d03ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d03ff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:45:51.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8ef460:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8ef460:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8ef460:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8ef460:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b47e6b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b94ed70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b94ed70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 1410fb6b3dbd69044b39b15a25d2305dc42779ea2118ecb1409e458c3280c771 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:1410fb6b3dbd69044b39b15a25d2305dc42779ea2118ecb1409e458c3280c771 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = c0e69dcaada6e614b11dbc009ee7092af791bb54d8293d4a1cdf921f5f6a0806 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c0e69dcaada6e614b11dbc009ee7092af791bb54d8293d4a1cdf921f5f6a0806 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=11, curl_handle=0x55e89b8e9f10 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1d4f412900 20 run: stack=0x55e89b8e7090 is io blocked 2026-03-20T11:45:51.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.649+0000 7f1ca97fa640 20 link_request req_data=0x55e89b8ea620 req_data->id=11, curl_handle=0x55e89b8e9f10 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8e5f60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8e5f60:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b479f30:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d4f412900 15 stack 0x55e89b482230 end 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is done 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d4f412900 20 collect(): s=0x55e89b8f22f0 stack=0x55e89b482230 is complete 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d4f412900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d4f412900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d4f412900 15 stack 0x55e89b8f22f0 end 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d4f412900 20 stack->operate() returned ret=-22 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d4f412900 20 run: stack=0x55e89b8f22f0 is done 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d4f412900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d4f412900 4 starting trim on bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8ed290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8ed290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d4f412900 20 enqueued request req=0x55e89b946940 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d4f412900 20 req: 0x55e89b946940 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d4f412900 20 run: stack=0x55e89b8f22f0 is io blocked 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d037f6640 20 dequeued request req=0x55e89b946940 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d037f6640 20 RGWWQ: empty 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d037f6640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 : miss 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d037f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d037f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d037f6640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 info.flags=0x17 2026-03-20T11:45:51.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d037f6640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 to cache LRU end 2026-03-20T11:45:51.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d037f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d037f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d037f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:45:51.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d037f6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-15 : miss 2026-03-20T11:45:51.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.685+0000 7f1d037f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b47abd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b47abd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b47abd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b47abd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b9461a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 15 stack 0x55e89b943030 end 2026-03-20T11:45:51.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 20 run: stack=0x55e89b943030 is done 2026-03-20T11:45:51.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 20 collect(): s=0x55e89b7ed970 stack=0x55e89b943030 is complete 2026-03-20T11:45:51.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 10 trimming bilogs for bucket=:dipepu-10[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15]) markers=[,00000000002.9.6,,,,,,,,,], shards=11 2026-03-20T11:45:51.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8eebc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 10 trimming bilog shard 1 of :dipepu-10[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15]) at marker 00000000002.9.6 2026-03-20T11:45:51.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b8f4ee0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b8f4ee0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15.1 2026-03-20T11:45:51.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 20 run: stack=0x55e89b943030 is io blocked 2026-03-20T11:45:51.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d037f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d037f6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-15 info.flags=0x0 2026-03-20T11:45:51.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d037f6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-15 to cache LRU end 2026-03-20T11:45:51.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d037f6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-15 : miss 2026-03-20T11:45:51.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d037f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d037f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d037f6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-15 info.flags=0x0 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d037f6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-15 to cache LRU end 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d037f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding source pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d037f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding dest pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d037f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d037f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d037f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8ed290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8ed290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8ed290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8ed290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b479f30:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 20 run: stack=0x55e89b8f22f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b946010:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b946010:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.686+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 1848d85fd7ef39cafe2ca3eeb54612015c825544bf2d40bcc3c7515de48268b2 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:1848d85fd7ef39cafe2ca3eeb54612015c825544bf2d40bcc3c7515de48268b2 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 5027cfc8780d26f7689447fefa4a637158e601742900cbd827308742e4fb509a 2026-03-20T11:45:51.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5027cfc8780d26f7689447fefa4a637158e601742900cbd827308742e4fb509a 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=12, curl_handle=0x55e89b534cc0 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is io blocked 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1ca97fa640 20 link_request req_data=0x55e89b5356e0 req_data->id=12, curl_handle=0x55e89b534cc0 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b8f4ee0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b8f4ee0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 15 stack 0x55e89b943030 end 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 20 run: stack=0x55e89b943030 is done 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8eebc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 20 trim: trimmed bucket instance dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 15 stack 0x55e89b7ed970 end 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is done 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 4 starting trim on bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8f4ee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8f4ee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 20 enqueued request req=0x55e89b8e7f40 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 20 req: 0x55e89b8e7f40 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is io blocked 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d02ff5640 20 dequeued request req=0x55e89b8e7f40 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d02ff5640 20 RGWWQ: empty 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d02ff5640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 : miss 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.687+0000 7f1d02ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.688+0000 7f1d02ff5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.688+0000 7f1d02ff5640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 info.flags=0x17 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.688+0000 7f1d02ff5640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 to cache LRU end 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.688+0000 7f1d02ff5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.688+0000 7f1d02ff5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.688+0000 7f1d02ff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.688+0000 7f1d02ff5640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-13 : miss 2026-03-20T11:45:51.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.688+0000 7f1d02ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.688+0000 7f1d02ff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.688+0000 7f1d02ff5640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-13 info.flags=0x0 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.688+0000 7f1d02ff5640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-13 to cache LRU end 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.688+0000 7f1d02ff5640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-13 : miss 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.688+0000 7f1d02ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d02ff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d02ff5640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-13 info.flags=0x0 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d02ff5640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-13 to cache LRU end 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d02ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18): adding source pipe: {s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d02ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18): adding dest pipe: {s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d02ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d02ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d02ff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8f4ee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8f4ee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8f4ee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8f4ee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b8f4ee0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b8ed290:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b8ed290:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.686 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.687 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.687 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.687 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.687 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.687 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.687 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.687 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 41d32b2f16f7a5a2e8e585d03cf357ed28cf97cb1248bb4bbf64d32284c587c5 2026-03-20T11:45:51.687 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.687 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.687 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.687 INFO:teuthology.orchestra.run.vm02.stderr:41d32b2f16f7a5a2e8e585d03cf357ed28cf97cb1248bb4bbf64d32284c587c5 2026-03-20T11:45:51.687 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.687 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.687 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.687 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.687 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = a956e9b2f02bf2d7d587f96057f2ad8bd8a8076a2aad2aea5aa729fcde2c87ec 2026-03-20T11:45:51.687 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a956e9b2f02bf2d7d587f96057f2ad8bd8a8076a2aad2aea5aa729fcde2c87ec 2026-03-20T11:45:51.687 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.687 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.687 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.687 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=13, curl_handle=0x55e89b47fb60 2026-03-20T11:45:51.687 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1d4f412900 20 run: stack=0x55e89b943030 is io blocked 2026-03-20T11:45:51.687 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.689+0000 7f1ca97fa640 20 link_request req_data=0x55e89b4802b0 req_data->id=13, curl_handle=0x55e89b47fb60 2026-03-20T11:45:51.688 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b47a5a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b47a5a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b94ed70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b47a5a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b94ed70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b47a5a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b94ed70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b8e5890:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 15 stack 0x55e89b947330 end 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 20 run: stack=0x55e89b947330 is done 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b94ed70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 20 collect(): s=0x55e89b8ea3f0 stack=0x55e89b947330 is complete 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 10 trimming bilogs for bucket=:dipepu-9[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14]) markers=[,00000000002.13.6,,,,,,,,,], shards=11 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b47e6b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 15 stack 0x55e89b8e7090 end 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 20 run: stack=0x55e89b8e7090 is done 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b532a90:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 10 trimming bilog shard 1 of :dipepu-9[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14]) at marker 00000000002.13.6 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 20 run: stack=0x55e89b8ea3f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 20 collect(): s=0x55e89b8e14b0 stack=0x55e89b8e7090 is complete 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 10 trimming bilogs for bucket=:dipepu-38[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43]) markers=[,,,,,,,,00000000001.38.6,,], shards=11 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b47adb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b47e6b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 10 trimming bilog shard 8 of :dipepu-38[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43]) at marker 00000000001.38.6 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b47adb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14.1 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 20 run: stack=0x55e89b947330 is io blocked 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b941eb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b941eb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43.8 2026-03-20T11:45:51.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.692+0000 7f1d4f412900 20 run: stack=0x55e89b8e7090 is io blocked 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b47adb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b47adb0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 15 stack 0x55e89b947330 end 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 run: stack=0x55e89b947330 is done 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b532a90:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 trim: trimmed bucket instance dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 15 stack 0x55e89b8ea3f0 end 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 run: stack=0x55e89b8ea3f0 is done 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 4 starting trim on bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b944490:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b944490:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 enqueued request req=0x55e89b94f4a0 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 req: 0x55e89b94f4a0 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d027f4640 20 dequeued request req=0x55e89b94f4a0 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d027f4640 20 RGWWQ: empty 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d027f4640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 : miss 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d027f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 run: stack=0x55e89b8ea3f0 is io blocked 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b941eb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b941eb0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 15 stack 0x55e89b8e7090 end 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 run: stack=0x55e89b8e7090 is done 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b47e6b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 trim: trimmed bucket instance dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 15 stack 0x55e89b8e14b0 end 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is done 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 4 starting trim on bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:45:51.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b47a5a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b47a5a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 enqueued request req=0x55e89b47c240 2026-03-20T11:45:51.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 req: 0x55e89b47c240 2026-03-20T11:45:51.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d01ff3640 20 dequeued request req=0x55e89b47c240 2026-03-20T11:45:51.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d01ff3640 20 RGWWQ: empty 2026-03-20T11:45:51.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d01ff3640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 : miss 2026-03-20T11:45:51.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d01ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is io blocked 2026-03-20T11:45:51.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d027f4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:51.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d027f4640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 info.flags=0x17 2026-03-20T11:45:51.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d027f4640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 to cache LRU end 2026-03-20T11:45:51.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d027f4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d027f4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d027f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:45:51.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d027f4640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-11 : miss 2026-03-20T11:45:51.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d027f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d01ff3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:51.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d01ff3640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 info.flags=0x17 2026-03-20T11:45:51.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d01ff3640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 to cache LRU end 2026-03-20T11:45:51.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d01ff3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d01ff3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d01ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:45:51.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d01ff3640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-12 : miss 2026-03-20T11:45:51.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.694+0000 7f1d01ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d01ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d01ff3640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-12 info.flags=0x0 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d01ff3640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-12 to cache LRU end 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d01ff3640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-12 : miss 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d01ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d027f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d027f4640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-11 info.flags=0x0 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d027f4640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-11 to cache LRU end 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d027f4640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-11 : miss 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d027f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d01ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d01ff3640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-12 info.flags=0x0 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d01ff3640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-12 to cache LRU end 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d01ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17): adding source pipe: {s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d01ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17): adding dest pipe: {s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d01ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d01ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d01ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d027f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d027f4640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-11 info.flags=0x0 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d027f4640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-11 to cache LRU end 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d027f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16): adding source pipe: {s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d027f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16): adding dest pipe: {s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d027f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d027f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d027f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b47a5a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b47a5a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b944490:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b47a5a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b944490:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b47a5a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b944490:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b944490:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b47e6b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b94ed70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b8e5890:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 run: stack=0x55e89b8ea3f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b94ed70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.693 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 25955dcecde8cc3fc3f84ab59f0080887d74910857a0ab6039f584aa51013b22 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:25955dcecde8cc3fc3f84ab59f0080887d74910857a0ab6039f584aa51013b22 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 30e1d625ed9d9c8f6340294b8cc05e0ae67e97916318eaee2c986b159bdf6ad9 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=30e1d625ed9d9c8f6340294b8cc05e0ae67e97916318eaee2c986b159bdf6ad9 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=14, curl_handle=0x55e89b8e9f10 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 run: stack=0x55e89b8e7090 is io blocked 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b9522b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b9522b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = c3f3863777db49e61c2aab8df31f86f876750002b1d407313739d9bbaf044745 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:c3f3863777db49e61c2aab8df31f86f876750002b1d407313739d9bbaf044745 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = bd6f9f354f344e22c3f165d3551e17d0bc499786dda0eed6475601ed2a2845be 2026-03-20T11:45:51.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=bd6f9f354f344e22c3f165d3551e17d0bc499786dda0eed6475601ed2a2845be 2026-03-20T11:45:51.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=15, curl_handle=0x55e89b47c780 2026-03-20T11:45:51.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1d4f412900 20 run: stack=0x55e89b947330 is io blocked 2026-03-20T11:45:51.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1ca97fa640 20 link_request req_data=0x55e89b8ea620 req_data->id=14, curl_handle=0x55e89b8e9f10 2026-03-20T11:45:51.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.695+0000 7f1ca97fa640 20 link_request req_data=0x7f1cf0017530 req_data->id=15, curl_handle=0x55e89b47c780 2026-03-20T11:45:51.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.730+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b946010:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.730+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b946010:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.730+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b946010:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.730+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b946010:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.730+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b479f30:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.730+0000 7f1d4f412900 15 stack 0x55e89b482230 end 2026-03-20T11:45:51.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.730+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is done 2026-03-20T11:45:51.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.730+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.730+0000 7f1d4f412900 20 collect(): s=0x55e89b8f22f0 stack=0x55e89b482230 is complete 2026-03-20T11:45:51.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.730+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.730+0000 7f1d4f412900 10 trimming bilogs for bucket=:dipepu-15[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20]) markers=[,,,,,,,,00000000007.20.6,,], shards=11 2026-03-20T11:45:51.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.730+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b945050:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.730+0000 7f1d4f412900 10 trimming bilog shard 8 of :dipepu-15[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20]) at marker 00000000007.20.6 2026-03-20T11:45:51.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.730+0000 7f1d4f412900 20 run: stack=0x55e89b8f22f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.730+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8ebda0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.730+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8ebda0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.730+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20.8 2026-03-20T11:45:51.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.730+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is io blocked 2026-03-20T11:45:51.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.731+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8ebda0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.731+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8ebda0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.731+0000 7f1d4f412900 15 stack 0x55e89b482230 end 2026-03-20T11:45:51.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.731+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.731+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is done 2026-03-20T11:45:51.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.731+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b945050:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.731+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.731+0000 7f1d4f412900 20 trim: trimmed bucket instance dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:45:51.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.731+0000 7f1d4f412900 15 stack 0x55e89b8f22f0 end 2026-03-20T11:45:51.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.731+0000 7f1d4f412900 20 run: stack=0x55e89b8f22f0 is done 2026-03-20T11:45:51.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.731+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.731+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.731+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.731+0000 7f1d4f412900 4 starting trim on bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:45:51.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.731+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8ebda0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.731+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8ebda0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.731+0000 7f1d4f412900 20 enqueued request req=0x55e89b946940 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.731+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.731+0000 7f1d4f412900 20 req: 0x55e89b946940 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.731+0000 7f1d017f2640 20 dequeued request req=0x55e89b946940 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.731+0000 7f1d017f2640 20 RGWWQ: empty 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.731+0000 7f1d017f2640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 : miss 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.731+0000 7f1d017f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.731+0000 7f1d4f412900 20 run: stack=0x55e89b8f22f0 is io blocked 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b8ed290:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b8ed290:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b8ed290:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b8ed290:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b8f4ee0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d4f412900 15 stack 0x55e89b943030 end 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d4f412900 20 run: stack=0x55e89b943030 is done 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d4f412900 20 collect(): s=0x55e89b7ed970 stack=0x55e89b943030 is complete 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d4f412900 10 trimming bilogs for bucket=:dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) markers=[,,,,,00000000003.29.10,,,00000000003.11.6,00000000011.37.6,], shards=11 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b94cb50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d4f412900 10 trimming bilog shard 5 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000003.29.10 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d4f412900 10 trimming bilog shard 8 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000003.11.6 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d4f412900 10 trimming bilog shard 9 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000011.37.6 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b8f4ee0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b94dbe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b8ed290:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b8f4ee0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.5 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d4f412900 20 run: stack=0x55e89b943030 is io blocked 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b94dbe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.8 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is io blocked 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b8ed290:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.9 2026-03-20T11:45:51.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is io blocked 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d017f2640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d017f2640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 info.flags=0x17 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d017f2640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 to cache LRU end 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d017f2640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d017f2640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d017f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d017f2640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-40 : miss 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d017f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d017f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d017f2640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-40 info.flags=0x0 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d017f2640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-40 to cache LRU end 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d017f2640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-40 : miss 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.732+0000 7f1d017f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d017f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d017f2640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-40 info.flags=0x0 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d017f2640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-40 to cache LRU end 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d017f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1): adding source pipe: {s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d017f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1): adding dest pipe: {s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d017f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d017f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d017f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8ebda0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8ebda0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8ebda0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8ebda0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b479f30:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 run: stack=0x55e89b8f22f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b952ce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b952ce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 83e085e30576301436a3f504b513b48e59bf819a4d4b92cd1dbe6b4cf601c401 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:83e085e30576301436a3f504b513b48e59bf819a4d4b92cd1dbe6b4cf601c401 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 3dcd5d5bc0a2ab9c5e6aea0b56c48ae0c2124e98339f8b676ca3bdf5ca5a6ab1 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=3dcd5d5bc0a2ab9c5e6aea0b56c48ae0c2124e98339f8b676ca3bdf5ca5a6ab1 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=16, curl_handle=0x55e89b47fb60 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 run: stack=0x55e89b47ccf0 is io blocked 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1ca97fa640 20 link_request req_data=0x55e89b4802b0 req_data->id=16, curl_handle=0x55e89b47fb60 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b8f4ee0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b8f4ee0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 15 stack 0x55e89b943030 end 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 run: stack=0x55e89b943030 is done 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b94cb50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b8ed290:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b8ed290:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 15 stack 0x55e89b946f40 end 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is done 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b94cb50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.733+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b94dbe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b94dbe0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d4f412900 15 stack 0x55e89b482230 end 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is done 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b94cb50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d4f412900 20 trim: trimmed bucket instance dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d4f412900 15 stack 0x55e89b7ed970 end 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is done 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d4f412900 4 starting trim on bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b9476b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b9476b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d4f412900 20 enqueued request req=0x55e89b8e1710 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d4f412900 20 req: 0x55e89b8e1710 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is io blocked 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d00ff1640 20 dequeued request req=0x55e89b8e1710 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d00ff1640 20 RGWWQ: empty 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d00ff1640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 : miss 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d00ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d00ff1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d00ff1640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 info.flags=0x17 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d00ff1640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 to cache LRU end 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d00ff1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d00ff1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d00ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d00ff1640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-18 : miss 2026-03-20T11:45:51.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.734+0000 7f1d00ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d00ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d00ff1640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-18 info.flags=0x0 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d00ff1640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-18 to cache LRU end 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d00ff1640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-18 : miss 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d00ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d00ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d00ff1640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-18 info.flags=0x0 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d00ff1640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-18 to cache LRU end 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d00ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23): adding source pipe: {s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d00ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23): adding dest pipe: {s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d00ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d00ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d00ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b9476b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b9476b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b9476b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b9476b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b951430:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8f66d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8f66d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = b00922d3433769ca5b9c59a1d1ed1a9456250d73d7d8e66a1b071b57f9be7abf 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:b00922d3433769ca5b9c59a1d1ed1a9456250d73d7d8e66a1b071b57f9be7abf 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = b3bfbd9ba3d5dcb961e5f8a14f0e1366cd6b24ebf5bbd64bbf8f9d37939696e2 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b3bfbd9ba3d5dcb961e5f8a14f0e1366cd6b24ebf5bbd64bbf8f9d37939696e2 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.734 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.734 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=17, curl_handle=0x55e89b534cc0 2026-03-20T11:45:51.734 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is io blocked 2026-03-20T11:45:51.734 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.735+0000 7f1ca97fa640 20 link_request req_data=0x55e89b5356e0 req_data->id=17, curl_handle=0x55e89b534cc0 2026-03-20T11:45:51.734 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b9522b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b9522b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b9522b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b9522b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b8e5890:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 15 stack 0x55e89b947330 end 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 run: stack=0x55e89b947330 is done 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 collect(): s=0x55e89b8ea3f0 stack=0x55e89b947330 is complete 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 10 trimming bilogs for bucket=:dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) markers=[00000000014.52.6,00000000008.34.6,00000000008.50.6,00000000008.32.6,00000000012.28.6,00000000010.31.6,00000000008.21.6,00000000010.27.6,00000000004.27.6,00000000008.22.6,00000000008.19.6], shards=11 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8f4fc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 10 trimming bilog shard 0 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000014.52.6 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 10 trimming bilog shard 1 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.34.6 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 10 trimming bilog shard 2 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.50.6 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 10 trimming bilog shard 3 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.32.6 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 10 trimming bilog shard 4 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000012.28.6 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 10 trimming bilog shard 5 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000010.31.6 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 10 trimming bilog shard 6 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.21.6 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 10 trimming bilog shard 7 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000010.27.6 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 10 trimming bilog shard 8 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000004.27.6 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 10 trimming bilog shard 9 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.22.6 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 10 trimming bilog shard 10 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.19.6 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 run: stack=0x55e89b8ea3f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b8ed290:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b947ff0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b9433f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b8e6be0:op=0x55e89b954310:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b951140:op=0x55e89b8f0ab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b955f10:op=0x55e89b945f30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b94e2a0:op=0x55e89b945050:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b8e0f70:op=0x55e89b960720:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b70e590:op=0x55e89b95fa30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b95dec0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b95c150:op=0x55e89b95ec60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b8ed290:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.0 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 run: stack=0x55e89b947330 is io blocked 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b947ff0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.1 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is io blocked 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b9433f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.2 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 run: stack=0x55e89b943030 is io blocked 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b8e6be0:op=0x55e89b954310:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.3 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 run: stack=0x55e89b8e6be0 is io blocked 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b951140:op=0x55e89b8f0ab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.4 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 run: stack=0x55e89b951140 is io blocked 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b955f10:op=0x55e89b945f30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.5 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 run: stack=0x55e89b955f10 is io blocked 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b94e2a0:op=0x55e89b945050:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.6 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 run: stack=0x55e89b94e2a0 is io blocked 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b8e0f70:op=0x55e89b960720:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.7 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 run: stack=0x55e89b8e0f70 is io blocked 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b70e590:op=0x55e89b95fa30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.8 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 run: stack=0x55e89b70e590 is io blocked 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b95dec0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.9 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 run: stack=0x55e89b95b890 is io blocked 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 cr:s=0x55e89b95c150:op=0x55e89b95ec60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.10 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.738+0000 7f1d4f412900 20 run: stack=0x55e89b95c150 is io blocked 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b94ed70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b94ed70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b94ed70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b94ed70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b47e6b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 15 stack 0x55e89b8e7090 end 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 run: stack=0x55e89b8e7090 is done 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 collect(): s=0x55e89b8e14b0 stack=0x55e89b8e7090 is complete 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 10 trimming bilogs for bucket=:dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) markers=[00000000014.44.6,00000000008.52.6,00000000008.40.6,00000000008.21.6,00000000012.26.6,00000000010.26.6,00000000008.35.6,00000000010.32.6,00000000004.38.6,00000000008.22.6,00000000008.23.6], shards=11 2026-03-20T11:45:51.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b47e6b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 10 trimming bilog shard 0 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000014.44.6 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 10 trimming bilog shard 1 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.52.6 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 10 trimming bilog shard 2 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.40.6 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 10 trimming bilog shard 3 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.21.6 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 10 trimming bilog shard 4 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000012.26.6 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 10 trimming bilog shard 5 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000010.26.6 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 10 trimming bilog shard 6 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.35.6 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 10 trimming bilog shard 7 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000010.32.6 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 10 trimming bilog shard 8 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000004.38.6 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 10 trimming bilog shard 9 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.22.6 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 10 trimming bilog shard 10 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.23.6 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b95d1e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b532c20:op=0x55e89b941eb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b982c70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b981f80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b9555d0:op=0x55e89b9802a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b981140:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b963930:op=0x55e89b9866f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b986ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b987970:op=0x55e89b983750:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b953730:op=0x55e89b983f50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b97ea90:op=0x55e89b97c590:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b95d1e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.0 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 run: stack=0x55e89b8e7090 is io blocked 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b532c20:op=0x55e89b941eb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.1 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 run: stack=0x55e89b532c20 is io blocked 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b982c70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.2 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is io blocked 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b981f80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.3 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 run: stack=0x55e89b97e430 is io blocked 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b9555d0:op=0x55e89b9802a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.4 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 run: stack=0x55e89b9555d0 is io blocked 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b981140:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.5 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is io blocked 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b963930:op=0x55e89b9866f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.6 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 run: stack=0x55e89b963930 is io blocked 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b986ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.7 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is io blocked 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b987970:op=0x55e89b983750:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.8 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 run: stack=0x55e89b987970 is io blocked 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b953730:op=0x55e89b983f50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.9 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 run: stack=0x55e89b953730 is io blocked 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 cr:s=0x55e89b97ea90:op=0x55e89b97c590:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.10 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.739+0000 7f1d4f412900 20 run: stack=0x55e89b97ea90 is io blocked 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.740+0000 7f1d4f412900 20 cr:s=0x55e89b95c150:op=0x55e89b95ec60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.740+0000 7f1d4f412900 20 cr:s=0x55e89b95c150:op=0x55e89b95ec60:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.740+0000 7f1d4f412900 15 stack 0x55e89b95c150 end 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.740+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.740+0000 7f1d4f412900 20 run: stack=0x55e89b95c150 is done 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.740+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8f4fc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.740+0000 7f1d4f412900 20 run: stack=0x55e89b8ea3f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.740+0000 7f1cf8fe1640 20 rgw rados thread: process(): notifying mdlog change, shard_id=2 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1cf8fe1640 20 rgw rados thread: cr:s=0x7f1d34024280:op=0x7f1d34057540:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1cf8fe1640 20 rgw rados thread: cr:s=0x7f1d34024280:op=0x7f1d34057540:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1cf8fe1640 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1cf8fe1640 20 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1cf8fe1640 20 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1cf8fe1640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:038966de9f6b9a901b20b4c6ca8b2a46009feebe031babc842d43690c0bc222b 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1cf8fe1640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]payload request hash = 038966de9f6b9a901b20b4c6ca8b2a46009feebe031babc842d43690c0bc222b 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1cf8fe1640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]canonical request = POST 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:notify=&rgwx-zonegroup=a&type=metadata 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:038966de9f6b9a901b20b4c6ca8b2a46009feebe031babc842d43690c0bc222b 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:038966de9f6b9a901b20b4c6ca8b2a46009feebe031babc842d43690c0bc222b 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1cf8fe1640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]canonical request hash = 7a3c1a5003711ac3c3e21dc897f0a3b43abd93f34143cdedb706ed8475655b37 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1cf8fe1640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:7a3c1a5003711ac3c3e21dc897f0a3b43abd93f34143cdedb706ed8475655b37 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1cf8fe1640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1cf8fe1640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1cf8fe1640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1cf8fe1640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1cf8fe1640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]generated signature = 4791d8dc432b858bc65586efe2be309c93a8153d737f513352b0cbfe55c212d0 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1cf8fe1640 20 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=4791d8dc432b858bc65586efe2be309c93a8153d737f513352b0cbfe55c212d0 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1cf8fe1640 20 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: 038966de9f6b9a901b20b4c6ca8b2a46009feebe031babc842d43690c0bc222b 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1cf8fe1640 20 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1cf8fe1640 20 sending request to http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1cf8fe1640 20 register_request mgr=0x55e89b46f9a8 req_data->id=0, curl_handle=0x55e89b8e9f10 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1cf8fe1640 20 run: stack=0x7f1d34024280 is io blocked 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1cf97e2640 20 link_request req_data=0x7f1d3405a900 req_data->id=0, curl_handle=0x55e89b8e9f10 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 cr:s=0x55e89b94e2a0:op=0x55e89b945050:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 cr:s=0x55e89b94e2a0:op=0x55e89b945050:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 15 stack 0x55e89b94e2a0 end 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 run: stack=0x55e89b94e2a0 is done 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8f4fc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 run: stack=0x55e89b8ea3f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b95dec0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b95dec0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 15 stack 0x55e89b95b890 end 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 run: stack=0x55e89b95b890 is done 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 cr:s=0x55e89b951140:op=0x55e89b8f0ab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 cr:s=0x55e89b951140:op=0x55e89b8f0ab0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 15 stack 0x55e89b951140 end 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 run: stack=0x55e89b951140 is done 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b947ff0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b947ff0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 15 stack 0x55e89b946f40 end 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is done 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8f4fc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8f4fc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8f4fc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 run: stack=0x55e89b8ea3f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b981f80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b981f80:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 15 stack 0x55e89b97e430 end 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 run: stack=0x55e89b97e430 is done 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b47e6b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b981140:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b981140:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 15 stack 0x55e89b942940 end 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is done 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b47e6b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b986ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b986ef0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 15 stack 0x55e89b945850 end 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is done 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b47e6b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.741+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.742+0000 7f1d4f412900 20 cr:s=0x55e89b953730:op=0x55e89b983f50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.742+0000 7f1d4f412900 20 cr:s=0x55e89b953730:op=0x55e89b983f50:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.742+0000 7f1d4f412900 15 stack 0x55e89b953730 end 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.742+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.742+0000 7f1d4f412900 20 run: stack=0x55e89b953730 is done 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.742+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b47e6b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.742+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.742+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b95d1e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.742+0000 7f1d4f412900 20 cr:s=0x55e89b8e7090:op=0x55e89b95d1e0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.742+0000 7f1d4f412900 15 stack 0x55e89b8e7090 end 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.742+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.742+0000 7f1d4f412900 20 run: stack=0x55e89b8e7090 is done 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.742+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b47e6b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.742+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.742+0000 7f1cf8fe1640 20 rgw rados thread: cr:s=0x7f1d34024280:op=0x7f1d34057540:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.742+0000 7f1cf8fe1640 20 rgw rados thread: cr:s=0x7f1d34024280:op=0x7f1d34057540:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.742+0000 7f1cf8fe1640 20 rgw rados thread: cr:s=0x7f1d34024280:op=0x7f1d34057540:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.742+0000 7f1cf8fe1640 20 rgw rados thread: cr:s=0x7f1d34024280:op=0x7f1d34057540:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.742+0000 7f1cf8fe1640 15 stack 0x7f1d34024280 end 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.742+0000 7f1cf8fe1640 20 run: stack=0x7f1d34024280 is done 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.742+0000 7f1d4f412900 20 cr:s=0x55e89b987970:op=0x55e89b983750:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.742+0000 7f1d4f412900 20 cr:s=0x55e89b987970:op=0x55e89b983750:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.742+0000 7f1d4f412900 15 stack 0x55e89b987970 end 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.742+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.742+0000 7f1d4f412900 20 run: stack=0x55e89b987970 is done 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.742+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b47e6b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.742+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 cr:s=0x55e89b97ea90:op=0x55e89b97c590:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 cr:s=0x55e89b97ea90:op=0x55e89b97c590:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 15 stack 0x55e89b97ea90 end 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 run: stack=0x55e89b97ea90 is done 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b47e6b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 cr:s=0x55e89b532c20:op=0x55e89b941eb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 cr:s=0x55e89b532c20:op=0x55e89b941eb0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 15 stack 0x55e89b532c20 end 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 run: stack=0x55e89b532c20 is done 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b47e6b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 cr:s=0x55e89b8e0f70:op=0x55e89b960720:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 cr:s=0x55e89b8e0f70:op=0x55e89b960720:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 15 stack 0x55e89b8e0f70 end 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 run: stack=0x55e89b8e0f70 is done 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8f4fc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 run: stack=0x55e89b8ea3f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 cr:s=0x55e89b9555d0:op=0x55e89b9802a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 cr:s=0x55e89b9555d0:op=0x55e89b9802a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 15 stack 0x55e89b9555d0 end 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 run: stack=0x55e89b9555d0 is done 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b47e6b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 cr:s=0x55e89b955f10:op=0x55e89b945f30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 cr:s=0x55e89b955f10:op=0x55e89b945f30:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 15 stack 0x55e89b955f10 end 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 run: stack=0x55e89b955f10 is done 2026-03-20T11:45:51.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8f4fc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.742 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 run: stack=0x55e89b8ea3f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.742 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 cr:s=0x55e89b8e6be0:op=0x55e89b954310:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.742 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 cr:s=0x55e89b8e6be0:op=0x55e89b954310:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.742 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 15 stack 0x55e89b8e6be0 end 2026-03-20T11:45:51.742 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.742 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 run: stack=0x55e89b8e6be0 is done 2026-03-20T11:45:51.742 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8f4fc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.742 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 run: stack=0x55e89b8ea3f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.742 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 cr:s=0x55e89b70e590:op=0x55e89b95fa30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.742 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 cr:s=0x55e89b70e590:op=0x55e89b95fa30:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.742 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 15 stack 0x55e89b70e590 end 2026-03-20T11:45:51.742 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.742 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 run: stack=0x55e89b70e590 is done 2026-03-20T11:45:51.742 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8f4fc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.742 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.743+0000 7f1d4f412900 20 run: stack=0x55e89b8ea3f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.742 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b8ed290:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.742 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 cr:s=0x55e89b947330:op=0x55e89b8ed290:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 15 stack 0x55e89b947330 end 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 run: stack=0x55e89b947330 is done 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8f4fc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 run: stack=0x55e89b8ea3f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 cr:s=0x55e89b963930:op=0x55e89b9866f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 cr:s=0x55e89b963930:op=0x55e89b9866f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 15 stack 0x55e89b963930 end 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 run: stack=0x55e89b963930 is done 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b47e6b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b9433f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 cr:s=0x55e89b943030:op=0x55e89b9433f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 15 stack 0x55e89b943030 end 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 run: stack=0x55e89b943030 is done 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8f4fc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 trim: trimmed bucket instance dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 15 stack 0x55e89b8ea3f0 end 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 run: stack=0x55e89b8ea3f0 is done 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 4 starting trim on bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b97f530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b97f530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 enqueued request req=0x55e89b9529a0 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 req: 0x55e89b9529a0 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is io blocked 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d007f0640 20 dequeued request req=0x55e89b9529a0 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d007f0640 20 RGWWQ: empty 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b982c70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b982c70:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 15 stack 0x55e89b97ec80 end 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is done 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b47e6b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 cr:s=0x55e89b8e14b0:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 trim: trimmed bucket instance dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 15 stack 0x55e89b8e14b0 end 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 run: stack=0x55e89b8e14b0 is done 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d007f0640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 : miss 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 4 starting trim on bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8f4f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8f4f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 enqueued request req=0x55e89b955790 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 req: 0x55e89b955790 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d007f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1cfffef640 20 dequeued request req=0x55e89b955790 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1cfffef640 20 RGWWQ: empty 2026-03-20T11:45:51.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1cfffef640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 : miss 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1cfffef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.744+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is io blocked 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d007f0640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d007f0640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 info.flags=0x17 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d007f0640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 to cache LRU end 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d007f0640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d007f0640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d007f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d007f0640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-50 : miss 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d007f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1cfffef640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1cfffef640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 info.flags=0x17 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1cfffef640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 to cache LRU end 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1cfffef640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1cfffef640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1cfffef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1cfffef640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-24 : miss 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1cfffef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d007f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d007f0640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-50 info.flags=0x0 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d007f0640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-50 to cache LRU end 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d007f0640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-50 : miss 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d007f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1cfffef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1cfffef640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-24 info.flags=0x0 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1cfffef640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-24 to cache LRU end 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1cfffef640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-24 : miss 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1cfffef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d007f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d007f0640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-50 info.flags=0x0 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d007f0640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-50 to cache LRU end 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1cfffef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1cfffef640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-24 info.flags=0x0 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d007f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10): adding source pipe: {s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1cfffef640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-24 to cache LRU end 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d007f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10): adding dest pipe: {s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1cfffef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29): adding source pipe: {s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1cfffef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29): adding dest pipe: {s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1cfffef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1cfffef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d007f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d007f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1cfffef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d007f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8f4f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8f4f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b97f530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8f4f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b97f530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8f4f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b97f530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b97f530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b47e6b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b952280:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b8f4f80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b952280:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 9c6bdde2d278ff42c1c28532297ff8637c53d19bbae4cc398acd3607d049cd4e 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:9c6bdde2d278ff42c1c28532297ff8637c53d19bbae4cc398acd3607d049cd4e 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 82f199819cbc48782c7467f1d756a43cd7adbe248d920cfa848285444ef42359 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=82f199819cbc48782c7467f1d756a43cd7adbe248d920cfa848285444ef42359 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=18, curl_handle=0x55e89b8e9f10 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is io blocked 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b97f530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b97f530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1ca97fa640 20 link_request req_data=0x55e89b8ea620 req_data->id=18, curl_handle=0x55e89b8e9f10 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.745 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.746 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.746 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.746 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.746 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.746 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.746 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = aba6f59f933c914be65e2f5d46791450253c773678697ae9567eb4e588b22d04 2026-03-20T11:45:51.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.746 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.746 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.746 INFO:teuthology.orchestra.run.vm02.stderr:aba6f59f933c914be65e2f5d46791450253c773678697ae9567eb4e588b22d04 2026-03-20T11:45:51.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 5695d17d37c0eaaf747389596aa54481c60fd5f0b14b90d2d65bf6d297a7e049 2026-03-20T11:45:51.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5695d17d37c0eaaf747389596aa54481c60fd5f0b14b90d2d65bf6d297a7e049 2026-03-20T11:45:51.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=19, curl_handle=0x55e89b47c780 2026-03-20T11:45:51.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1d4f412900 20 run: stack=0x55e89b97e430 is io blocked 2026-03-20T11:45:51.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.745+0000 7f1ca97fa640 20 link_request req_data=0x7f1cf0017530 req_data->id=19, curl_handle=0x55e89b47c780 2026-03-20T11:45:51.746 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:51.745+0000 7fcfd41f6640 -1 req 11040634951289948192 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:51.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.776+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b952ce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.776+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b952ce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.776+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b952ce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.776+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b952ce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.776+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b479f30:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.776+0000 7f1d4f412900 15 stack 0x55e89b47ccf0 end 2026-03-20T11:45:51.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.776+0000 7f1d4f412900 20 run: stack=0x55e89b47ccf0 is done 2026-03-20T11:45:51.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.776+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.776+0000 7f1d4f412900 20 collect(): s=0x55e89b8f22f0 stack=0x55e89b47ccf0 is complete 2026-03-20T11:45:51.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.776+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.776+0000 7f1d4f412900 10 trimming bilogs for bucket=:dipepu-40[4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1]) markers=[,,,,,00000000001.103.6,,,,,], shards=11 2026-03-20T11:45:51.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.776+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b9802a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.776+0000 7f1d4f412900 10 trimming bilog shard 5 of :dipepu-40[4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1]) at marker 00000000001.103.6 2026-03-20T11:45:51.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.776+0000 7f1d4f412900 20 run: stack=0x55e89b8f22f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.776+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b945050:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.776+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b945050:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.776+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1.5 2026-03-20T11:45:51.773 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.776+0000 7f1d4f412900 20 run: stack=0x55e89b47ccf0 is io blocked 2026-03-20T11:45:51.774 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.778+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8f66d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.775 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.778+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8f66d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.775 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.778+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8f66d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.775 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.778+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8f66d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.775 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.778+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b951430:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.775 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.778+0000 7f1d4f412900 15 stack 0x55e89b482230 end 2026-03-20T11:45:51.775 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.778+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is done 2026-03-20T11:45:51.775 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.778+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.775 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.778+0000 7f1d4f412900 20 collect(): s=0x55e89b7ed970 stack=0x55e89b482230 is complete 2026-03-20T11:45:51.775 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.778+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.775 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.778+0000 7f1d4f412900 10 trimming bilogs for bucket=:dipepu-18[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23]) markers=[,,,,,,,,00000000005.51.12,,], shards=11 2026-03-20T11:45:51.775 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.778+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b95f9b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.775 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.778+0000 7f1d4f412900 10 trimming bilog shard 8 of :dipepu-18[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23]) at marker 00000000005.51.12 2026-03-20T11:45:51.775 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.778+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.775 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.778+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b95ec00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.775 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.778+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b95ec00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.775 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.778+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23.8 2026-03-20T11:45:51.775 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.778+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is io blocked 2026-03-20T11:45:51.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b952280:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b952280:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:51.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b47e6b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 15 stack 0x55e89b942940 end 2026-03-20T11:45:51.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is done 2026-03-20T11:45:51.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b97f530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 collect(): s=0x55e89b97ec80 stack=0x55e89b942940 is complete 2026-03-20T11:45:51.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:51.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:51.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:51.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 15 stack 0x55e89b97ec80 end 2026-03-20T11:45:51.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 stack->operate() returned ret=-22 2026-03-20T11:45:51.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is done 2026-03-20T11:45:51.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b97f530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:51.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b97f530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 4 starting trim on bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:45:51.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b97f530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b8f4f80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 15 stack 0x55e89b97e430 end 2026-03-20T11:45:51.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 run: stack=0x55e89b97e430 is done 2026-03-20T11:45:51.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 enqueued request req=0x55e89b9529a0 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 req: 0x55e89b9529a0 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is io blocked 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 collect(): s=0x55e89b945850 stack=0x55e89b97e430 is complete 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 15 stack 0x55e89b945850 end 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 stack->operate() returned ret=-22 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is done 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 4 starting trim on bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b9816f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b9816f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1cff7ee640 20 dequeued request req=0x55e89b9529a0 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1cff7ee640 20 RGWWQ: empty 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1cff7ee640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 : miss 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 enqueued request req=0x55e89b97df60 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 req: 0x55e89b97df60 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is io blocked 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1cff7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1cfefed640 20 dequeued request req=0x55e89b97df60 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1cfefed640 20 RGWWQ: empty 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1cfefed640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 : miss 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1cfefed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1cff7ee640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1cff7ee640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 info.flags=0x17 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1cff7ee640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 to cache LRU end 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1cff7ee640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1cff7ee640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1cff7ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1cff7ee640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-56 : miss 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.787+0000 7f1cff7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cfefed640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:51.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cfefed640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 info.flags=0x17 2026-03-20T11:45:51.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cfefed640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 to cache LRU end 2026-03-20T11:45:51.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cfefed640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cfefed640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cfefed640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:45:51.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cfefed640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-27 : miss 2026-03-20T11:45:51.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cfefed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cff7ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cff7ee640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-56 info.flags=0x0 2026-03-20T11:45:51.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cfefed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cff7ee640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-56 to cache LRU end 2026-03-20T11:45:51.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cff7ee640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-56 : miss 2026-03-20T11:45:51.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cff7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cfefed640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-27 info.flags=0x0 2026-03-20T11:45:51.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cfefed640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-27 to cache LRU end 2026-03-20T11:45:51.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cfefed640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-27 : miss 2026-03-20T11:45:51.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cfefed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cfefed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cfefed640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-27 info.flags=0x0 2026-03-20T11:45:51.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cfefed640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-27 to cache LRU end 2026-03-20T11:45:51.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cfefed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32): adding source pipe: {s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cfefed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32): adding dest pipe: {s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cfefed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cfefed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cfefed640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:45:51.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b9816f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b9816f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b9816f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b9816f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b951430:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cff7ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cff7ee640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-56 info.flags=0x0 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cff7ee640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-56 to cache LRU end 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cff7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16): adding source pipe: {s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cff7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16): adding dest pipe: {s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cff7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cff7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1cff7ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 248fa7c7bb779994f1a08e4d68180fda8925b9eede158a5ca899b74ff6bd2ef3 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:248fa7c7bb779994f1a08e4d68180fda8925b9eede158a5ca899b74ff6bd2ef3 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 2635a401cd717ee5723010b40f57688f9856a654babc9bb8f94d3c97eca38af8 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2635a401cd717ee5723010b40f57688f9856a654babc9bb8f94d3c97eca38af8 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=20, curl_handle=0x55e89b47c780 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 run: stack=0x55e89b97e430 is io blocked 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b8f6690:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1ca97fa640 20 link_request req_data=0x7f1cf0017530 req_data->id=20, curl_handle=0x55e89b47c780 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = ca05c620e7d62e43fd1fcf9c380b92f3caefcf9c0b8fc077df4f8d6be34184ae 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:ca05c620e7d62e43fd1fcf9c380b92f3caefcf9c0b8fc077df4f8d6be34184ae 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 9a4e76cb1b29c19b6516bd83ffd7898831c9dc583528e3e4f6110655ea20fe1a 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=9a4e76cb1b29c19b6516bd83ffd7898831c9dc583528e3e4f6110655ea20fe1a 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=21, curl_handle=0x55e89b8e9f10 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is io blocked 2026-03-20T11:45:51.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.788+0000 7f1ca97fa640 20 link_request req_data=0x55e89b8ea620 req_data->id=21, curl_handle=0x55e89b8e9f10 2026-03-20T11:45:51.788 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:51.788+0000 7fcfdf20c640 -1 req 13627316998325622852 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:51.788 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:51.788+0000 7fd071b31640 -1 req 15385671112713187262 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:51.788 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:51.790+0000 7fd073334640 -1 req 10971812307743097822 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b945050:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b945050:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1d4f412900 15 stack 0x55e89b47ccf0 end 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1d4f412900 20 run: stack=0x55e89b47ccf0 is done 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b9802a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1d4f412900 20 trim: trimmed bucket instance dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1d4f412900 15 stack 0x55e89b8f22f0 end 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1d4f412900 20 run: stack=0x55e89b8f22f0 is done 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1d4f412900 4 starting trim on bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1d4f412900 20 enqueued request req=0x55e89b946940 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1d4f412900 20 req: 0x55e89b946940 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1d4f412900 20 run: stack=0x55e89b8f22f0 is io blocked 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1cfe7ec640 20 dequeued request req=0x55e89b946940 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1cfe7ec640 20 RGWWQ: empty 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1cfe7ec640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 : miss 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1cfe7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1cfe7ec640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=314 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1cfe7ec640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 info.flags=0x17 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1cfe7ec640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 to cache LRU end 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1cfe7ec640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1cfe7ec640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1cfe7ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1cfe7ec640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-44 : miss 2026-03-20T11:45:51.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1cfe7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1cfe7ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1cfe7ec640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-44 info.flags=0x0 2026-03-20T11:45:51.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1cfe7ec640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-44 to cache LRU end 2026-03-20T11:45:51.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1cfe7ec640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-44 : miss 2026-03-20T11:45:51.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.790+0000 7f1cfe7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1cfe7ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1cfe7ec640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-44 info.flags=0x0 2026-03-20T11:45:51.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1cfe7ec640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-44 to cache LRU end 2026-03-20T11:45:51.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1cfe7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4): adding source pipe: {s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1cfe7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4): adding dest pipe: {s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1cfe7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1cfe7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1cfe7ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:45:51.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b8f4f80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 20 run: stack=0x55e89b8f22f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b952c10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b952c10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.794 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.794 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.794 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 25f64aff38e74307d95c458de86200e92bfb884176b3de580b47f243813e4516 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr:25f64aff38e74307d95c458de86200e92bfb884176b3de580b47f243813e4516 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 1fc9b4fbe5ee134ab791d5448d10aab7673bdeb62672058d5045d4f6f6c725aa 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=1fc9b4fbe5ee134ab791d5448d10aab7673bdeb62672058d5045d4f6f6c725aa 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=22, curl_handle=0x55e89b534cc0 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1d4f412900 20 run: stack=0x55e89b47ccf0 is io blocked 2026-03-20T11:45:51.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.791+0000 7f1ca97fa640 20 link_request req_data=0x55e89b5356e0 req_data->id=22, curl_handle=0x55e89b534cc0 2026-03-20T11:45:51.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b95ec00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b95ec00:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1d4f412900 15 stack 0x55e89b482230 end 2026-03-20T11:45:51.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is done 2026-03-20T11:45:51.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b95f9b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1d4f412900 20 trim: trimmed bucket instance dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:45:51.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1d4f412900 15 stack 0x55e89b7ed970 end 2026-03-20T11:45:51.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is done 2026-03-20T11:45:51.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1d4f412900 4 starting trim on bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:45:51.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1d4f412900 20 enqueued request req=0x55e89b8e1710 2026-03-20T11:45:51.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1d4f412900 20 req: 0x55e89b8e1710 2026-03-20T11:45:51.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is io blocked 2026-03-20T11:45:51.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1cfdfeb640 20 dequeued request req=0x55e89b8e1710 2026-03-20T11:45:51.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1cfdfeb640 20 RGWWQ: empty 2026-03-20T11:45:51.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1cfdfeb640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 : miss 2026-03-20T11:45:51.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1cfdfeb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1cfdfeb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:51.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1cfdfeb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 info.flags=0x17 2026-03-20T11:45:51.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1cfdfeb640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 to cache LRU end 2026-03-20T11:45:51.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1cfdfeb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1cfdfeb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1cfdfeb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1cfdfeb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-17 : miss 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.806+0000 7f1cfdfeb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1cfdfeb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1cfdfeb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-17 info.flags=0x0 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1cfdfeb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-17 to cache LRU end 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1cfdfeb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-17 : miss 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1cfdfeb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1cfdfeb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1cfdfeb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-17 info.flags=0x0 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1cfdfeb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-17 to cache LRU end 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1cfdfeb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22): adding source pipe: {s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1cfdfeb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22): adding dest pipe: {s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1cfdfeb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1cfdfeb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1cfdfeb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b952280:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b949e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b949e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = db8cbf7fc1ccbb624a70533e512f1b47c4e35c6d2cc24446f8f518c6e09252b2 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:db8cbf7fc1ccbb624a70533e512f1b47c4e35c6d2cc24446f8f518c6e09252b2 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 8e1dfd6bc5346f311ee54f2f70ada89ec59259d4f534f9cec98154e0a7ebbf79 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=8e1dfd6bc5346f311ee54f2f70ada89ec59259d4f534f9cec98154e0a7ebbf79 2026-03-20T11:45:51.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.819 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.819 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.819 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=23, curl_handle=0x55e89b47fb60 2026-03-20T11:45:51.819 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is io blocked 2026-03-20T11:45:51.819 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.807+0000 7f1ca97fa640 20 link_request req_data=0x55e89b4802b0 req_data->id=23, curl_handle=0x55e89b47fb60 2026-03-20T11:45:51.827 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:51.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b951430:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 15 stack 0x55e89b97e430 end 2026-03-20T11:45:51.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 run: stack=0x55e89b97e430 is done 2026-03-20T11:45:51.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:51.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 collect(): s=0x55e89b945850 stack=0x55e89b97e430 is complete 2026-03-20T11:45:51.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:51.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:51.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:51.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 15 stack 0x55e89b945850 end 2026-03-20T11:45:51.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 stack->operate() returned ret=-22 2026-03-20T11:45:51.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is done 2026-03-20T11:45:51.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b8f6690:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 15 stack 0x55e89b942940 end 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is done 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 collect(): s=0x55e89b97ec80 stack=0x55e89b942940 is complete 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 15 stack 0x55e89b97ec80 end 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 stack->operate() returned ret=-22 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is done 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 4 starting trim on bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b948820:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 4 starting trim on bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b948820:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 enqueued request req=0x55e89b8e7b20 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 req: 0x55e89b8e7b20 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is io blocked 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b97d7b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b97d7b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 enqueued request req=0x55e89b955260 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 req: 0x55e89b8e7b20 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 req: 0x55e89b955260 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is io blocked 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1cfcfe9640 20 dequeued request req=0x55e89b8e7b20 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1cfcfe9640 20 RGWWQ: 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1cfcfe9640 20 req: 0x55e89b955260 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1cfcfe9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 : miss 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1cfcfe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1cfd7ea640 20 dequeued request req=0x55e89b955260 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1cfd7ea640 20 RGWWQ: empty 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1cfd7ea640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 : miss 2026-03-20T11:45:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1cfd7ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.829 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:51.831+0000 7fd071330640 -1 req 17787223912458315275 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:51.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1cfcfe9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:51.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1cfcfe9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 info.flags=0x17 2026-03-20T11:45:51.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1cfcfe9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 to cache LRU end 2026-03-20T11:45:51.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1cfcfe9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1cfcfe9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1cfcfe9640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:45:51.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1cfcfe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:45:51.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1cfcfe9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-46 : miss 2026-03-20T11:45:51.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1cfcfe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1cfd7ea640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:51.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1cfd7ea640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 info.flags=0x17 2026-03-20T11:45:51.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1cfd7ea640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 to cache LRU end 2026-03-20T11:45:51.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1cfd7ea640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1cfd7ea640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.830+0000 7f1cfd7ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:45:51.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1cfd7ea640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-37 : miss 2026-03-20T11:45:51.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1cfd7ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1cfd7ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1cfd7ea640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-37 info.flags=0x0 2026-03-20T11:45:51.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1cfd7ea640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-37 to cache LRU end 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1cfd7ea640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-37 : miss 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1cfd7ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1cfcfe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1cfcfe9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-46 info.flags=0x0 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1cfcfe9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-46 to cache LRU end 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1cfcfe9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-46 : miss 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1cfcfe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1cfd7ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1cfd7ea640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-37 info.flags=0x0 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1cfd7ea640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-37 to cache LRU end 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1cfd7ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42): adding source pipe: {s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1cfd7ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42): adding dest pipe: {s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1cfd7ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1cfd7ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1cfd7ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1cfcfe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1cfcfe9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-46 info.flags=0x0 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1cfcfe9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-46 to cache LRU end 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b97d7b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b97d7b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b97d7b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b97d7b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1cfcfe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6): adding source pipe: {s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1cfcfe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6): adding dest pipe: {s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b8f6690:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1cfcfe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1cfcfe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1cfcfe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = a10ac4daf04d64e2da17394c00444828f8d7113824a66c7c69d9f5a435fd7999 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:a10ac4daf04d64e2da17394c00444828f8d7113824a66c7c69d9f5a435fd7999 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 1aac93b6ceff1d68243f6a4fc791422afa39d53f63162f55bfbf6e133f62380c 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=1aac93b6ceff1d68243f6a4fc791422afa39d53f63162f55bfbf6e133f62380c 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=24, curl_handle=0x55e89b8e9f10 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is io blocked 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b948820:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b948820:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b948820:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b948820:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b951430:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 514e252102f1b25c1f941b9ce86cce69f29c54e7f94cd97613e08fc0000026fe 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:514e252102f1b25c1f941b9ce86cce69f29c54e7f94cd97613e08fc0000026fe 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 41802b8fcb682554072dc1dd7b7d7e0eca91773e99129856b5e92971d762ffb2 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=41802b8fcb682554072dc1dd7b7d7e0eca91773e99129856b5e92971d762ffb2 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=25, curl_handle=0x55e89b47c780 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1d4f412900 20 run: stack=0x55e89b97e430 is io blocked 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1ca97fa640 20 link_request req_data=0x55e89b8ea620 req_data->id=24, curl_handle=0x55e89b8e9f10 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.831+0000 7f1ca97fa640 20 link_request req_data=0x7f1cf0017530 req_data->id=25, curl_handle=0x55e89b47c780 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b952c10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b952c10:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b8f4f80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1d4f412900 15 stack 0x55e89b47ccf0 end 2026-03-20T11:45:51.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1d4f412900 20 run: stack=0x55e89b47ccf0 is done 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1d4f412900 20 collect(): s=0x55e89b8f22f0 stack=0x55e89b47ccf0 is complete 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1d4f412900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1d4f412900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1d4f412900 15 stack 0x55e89b8f22f0 end 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1d4f412900 20 stack->operate() returned ret=-22 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1d4f412900 20 run: stack=0x55e89b8f22f0 is done 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1d4f412900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1d4f412900 4 starting trim on bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b948820:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b948820:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1d4f412900 20 enqueued request req=0x55e89b946940 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1d4f412900 20 req: 0x55e89b946940 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1d4f412900 20 run: stack=0x55e89b8f22f0 is io blocked 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1cfc7e8640 20 dequeued request req=0x55e89b946940 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1cfc7e8640 20 RGWWQ: empty 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1cfc7e8640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 : miss 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1cfc7e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1cfc7e8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1cfc7e8640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 info.flags=0x17 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1cfc7e8640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 to cache LRU end 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1cfc7e8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1cfc7e8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1cfc7e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1cfc7e8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-1 : miss 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1cfc7e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1cfc7e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1cfc7e8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-1 info.flags=0x0 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1cfc7e8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-1 to cache LRU end 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1cfc7e8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-1 : miss 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.832+0000 7f1cfc7e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1cfc7e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1cfc7e8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-1 info.flags=0x0 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1cfc7e8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-1 to cache LRU end 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1cfc7e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1): adding source pipe: {s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1cfc7e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1): adding dest pipe: {s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1cfc7e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1cfc7e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1cfc7e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b948820:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b948820:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b948820:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b948820:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b8f4f80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 20 run: stack=0x55e89b8f22f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b981f20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b981f20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 201575a3afcbdedffebe33fe3fd1707b954e2b5e98b1269957f9ca98cdcba087 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:201575a3afcbdedffebe33fe3fd1707b954e2b5e98b1269957f9ca98cdcba087 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = a188758dd9641984fd2935becbe4bbd75815aa77d0853f7797224105180eb2c0 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a188758dd9641984fd2935becbe4bbd75815aa77d0853f7797224105180eb2c0 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=26, curl_handle=0x55e89b534cc0 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1d4f412900 20 run: stack=0x55e89b47ccf0 is io blocked 2026-03-20T11:45:51.834 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.833+0000 7f1ca97fa640 20 link_request req_data=0x55e89b47ced0 req_data->id=26, curl_handle=0x55e89b534cc0 2026-03-20T11:45:51.834 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:51.833+0000 7fcfd61fa640 -1 req 6776774062180048862 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:51.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.850+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b949e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.850+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b949e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.850+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b949e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.850+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b949e10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.850+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b952280:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.850+0000 7f1d4f412900 15 stack 0x55e89b482230 end 2026-03-20T11:45:51.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.850+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is done 2026-03-20T11:45:51.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.850+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.850+0000 7f1d4f412900 20 collect(): s=0x55e89b7ed970 stack=0x55e89b482230 is complete 2026-03-20T11:45:51.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.850+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.850+0000 7f1d4f412900 10 trimming bilogs for bucket=:dipepu-17[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22]) markers=[,,,,,,,,00000000005.46.18,,], shards=11 2026-03-20T11:45:51.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.850+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b981060:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.850+0000 7f1d4f412900 10 trimming bilog shard 8 of :dipepu-17[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22]) at marker 00000000005.46.18 2026-03-20T11:45:51.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.850+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.850+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b945050:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.850+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b945050:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.850+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22.8 2026-03-20T11:45:51.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.850+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is io blocked 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b945050:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b945050:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1d4f412900 15 stack 0x55e89b482230 end 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is done 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b981060:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1d4f412900 20 trim: trimmed bucket instance dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1d4f412900 15 stack 0x55e89b7ed970 end 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is done 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1d4f412900 4 starting trim on bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b945050:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b945050:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1d4f412900 20 enqueued request req=0x55e89b8e1710 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1d4f412900 20 req: 0x55e89b8e1710 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is io blocked 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1cfbfe7640 20 dequeued request req=0x55e89b8e1710 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1cfbfe7640 20 RGWWQ: empty 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1cfbfe7640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 : miss 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1cfbfe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1cfbfe7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1cfbfe7640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 info.flags=0x17 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1cfbfe7640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 to cache LRU end 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1cfbfe7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1cfbfe7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1cfbfe7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1cfbfe7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-7 : miss 2026-03-20T11:45:51.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.864+0000 7f1cfbfe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.865+0000 7f1cfbfe7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.865+0000 7f1cfbfe7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-7 info.flags=0x0 2026-03-20T11:45:51.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.865+0000 7f1cfbfe7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-7 to cache LRU end 2026-03-20T11:45:51.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.865+0000 7f1cfbfe7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-7 : miss 2026-03-20T11:45:51.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.865+0000 7f1cfbfe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.865+0000 7f1cfbfe7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.865+0000 7f1cfbfe7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-7 info.flags=0x0 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.865+0000 7f1cfbfe7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-7 to cache LRU end 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.865+0000 7f1cfbfe7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12): adding source pipe: {s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.865+0000 7f1cfbfe7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12): adding dest pipe: {s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.865+0000 7f1cfbfe7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.865+0000 7f1cfbfe7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.865+0000 7f1cfbfe7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.865+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b945050:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.865+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b945050:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.865+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b945050:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.865+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b945050:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.865+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.865+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b949e10:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.865+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.865+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.865+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b479f30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.865+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b479f30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.865+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.865+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.865+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.865+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.866+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.866+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.866+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 68e88f1fd0a709c0458415ddce8a5abf4e5a1e499bee43e83dad9a476f91a616 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.866+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:68e88f1fd0a709c0458415ddce8a5abf4e5a1e499bee43e83dad9a476f91a616 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.866+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.866+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.866+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.866+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.866+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 50011769a2f6cd362f964bd750403d7ce1d400070b7d3ea5f9020d2e72858b49 2026-03-20T11:45:51.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.866+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=50011769a2f6cd362f964bd750403d7ce1d400070b7d3ea5f9020d2e72858b49 2026-03-20T11:45:51.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.866+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.866+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.866+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.866+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=27, curl_handle=0x55e89b47fb60 2026-03-20T11:45:51.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.866+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is io blocked 2026-03-20T11:45:51.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.866+0000 7f1ca97fa640 20 link_request req_data=0x55e89b47c820 req_data->id=27, curl_handle=0x55e89b47fb60 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b951430:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1d4f412900 15 stack 0x55e89b97e430 end 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1d4f412900 20 run: stack=0x55e89b97e430 is done 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1d4f412900 20 collect(): s=0x55e89b945850 stack=0x55e89b97e430 is complete 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1d4f412900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1d4f412900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1d4f412900 15 stack 0x55e89b945850 end 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1d4f412900 20 stack->operate() returned ret=-22 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is done 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1d4f412900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1d4f412900 4 starting trim on bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b945050:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b945050:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1d4f412900 20 enqueued request req=0x55e89b8e7b20 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1d4f412900 20 req: 0x55e89b8e7b20 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is io blocked 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1cfb7e6640 20 dequeued request req=0x55e89b8e7b20 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1cfb7e6640 20 RGWWQ: empty 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1cfb7e6640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 : miss 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1cfb7e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1cfb7e6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1cfb7e6640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 info.flags=0x17 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1cfb7e6640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 to cache LRU end 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1cfb7e6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1cfb7e6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1cfb7e6640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1cfb7e6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1cfb7e6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-48 : miss 2026-03-20T11:45:51.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.873+0000 7f1cfb7e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1cfb7e6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1cfb7e6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-48 info.flags=0x0 2026-03-20T11:45:51.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1cfb7e6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-48 to cache LRU end 2026-03-20T11:45:51.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1cfb7e6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-48 : miss 2026-03-20T11:45:51.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1cfb7e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1cfb7e6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1cfb7e6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-48 info.flags=0x0 2026-03-20T11:45:51.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1cfb7e6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-48 to cache LRU end 2026-03-20T11:45:51.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1cfb7e6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8): adding source pipe: {s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1cfb7e6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8): adding dest pipe: {s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1cfb7e6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1cfb7e6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1cfb7e6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b945050:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b945050:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b945050:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b945050:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b951430:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 992ff7273a5808659720c73e88eecdee1e400a82d2fdbe44098a583248fffaa4 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:992ff7273a5808659720c73e88eecdee1e400a82d2fdbe44098a583248fffaa4 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 163a2f86a8090b0299e5072509e6ea7472711cda1653d5ec904fa8fc45af7490 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=163a2f86a8090b0299e5072509e6ea7472711cda1653d5ec904fa8fc45af7490 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=28, curl_handle=0x55e89b47c780 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1d4f412900 20 run: stack=0x55e89b97e430 is io blocked 2026-03-20T11:45:51.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.874+0000 7f1ca97fa640 20 link_request req_data=0x55e89b94d8e0 req_data->id=28, curl_handle=0x55e89b47c780 2026-03-20T11:45:51.873 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:51.874+0000 7fd06b324640 -1 req 12780263194588746396 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b8f6690:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 15 stack 0x55e89b942940 end 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is done 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 collect(): s=0x55e89b97ec80 stack=0x55e89b942940 is complete 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 10 trimming bilogs for bucket=:dipepu-37[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42]) markers=[,,,,,,,,,,00000000001.20.6], shards=11 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8f6690:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 10 trimming bilog shard 10 of :dipepu-37[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42]) at marker 00000000001.20.6 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b95ec00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b95ec00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42.10 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is io blocked 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b981f20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b981f20:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b8f4f80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 15 stack 0x55e89b47ccf0 end 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 run: stack=0x55e89b47ccf0 is done 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 collect(): s=0x55e89b8f22f0 stack=0x55e89b47ccf0 is complete 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b8e45c0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 15 stack 0x55e89b8f22f0 end 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 stack->operate() returned ret=-22 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 run: stack=0x55e89b8f22f0 is done 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b963f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 4 starting trim on bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:45:51.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b9581c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b9581c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 enqueued request req=0x55e89b946940 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 req: 0x55e89b946940 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d12ffd640 20 dequeued request req=0x55e89b946940 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d12ffd640 20 RGWWQ: empty 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d12ffd640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 : miss 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d12ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.875+0000 7f1d4f412900 20 run: stack=0x55e89b8f22f0 is io blocked 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.876+0000 7f1d12ffd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.876+0000 7f1d12ffd640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 info.flags=0x17 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.876+0000 7f1d12ffd640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 to cache LRU end 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.876+0000 7f1d12ffd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.876+0000 7f1d12ffd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.876+0000 7f1d12ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.876+0000 7f1d12ffd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-59 : miss 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.876+0000 7f1d12ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.876+0000 7f1d12ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.876+0000 7f1d12ffd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-59 info.flags=0x0 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.876+0000 7f1d12ffd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-59 to cache LRU end 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.876+0000 7f1d12ffd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-59 : miss 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.876+0000 7f1d12ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b95ec00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b95ec00:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 15 stack 0x55e89b942940 end 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is done 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8f6690:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 trim: trimmed bucket instance dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 15 stack 0x55e89b97ec80 end 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is done 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 4 starting trim on bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b95e3f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b95e3f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 enqueued request req=0x55e89b955260 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 req: 0x55e89b955260 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is io blocked 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d127fc640 20 dequeued request req=0x55e89b955260 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d127fc640 20 RGWWQ: empty 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d127fc640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 : miss 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d127fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d12ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d12ffd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-59 info.flags=0x0 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d12ffd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-59 to cache LRU end 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d12ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19): adding source pipe: {s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d12ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19): adding dest pipe: {s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d12ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d12ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d12ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b9581c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b9581c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b9581c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b9581c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b963f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b8f6690:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b963f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 run: stack=0x55e89b8f22f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b957b40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b957b40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 806f640835b5b860393acc28c6f14161032bc23a5e24b4d551316f2d4de59342 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.875 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.876 INFO:teuthology.orchestra.run.vm02.stderr:806f640835b5b860393acc28c6f14161032bc23a5e24b4d551316f2d4de59342 2026-03-20T11:45:51.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d127fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:51.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d127fc640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 info.flags=0x17 2026-03-20T11:45:51.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d127fc640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 to cache LRU end 2026-03-20T11:45:51.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d127fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d127fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d127fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:45:51.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = fb21804d89fd4bbb83b9bfae7e783b2b5ee94d3c1695a18b95d335d5b9a1a461 2026-03-20T11:45:51.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=fb21804d89fd4bbb83b9bfae7e783b2b5ee94d3c1695a18b95d335d5b9a1a461 2026-03-20T11:45:51.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d127fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-21 : miss 2026-03-20T11:45:51.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d127fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=29, curl_handle=0x55e89b534cc0 2026-03-20T11:45:51.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is io blocked 2026-03-20T11:45:51.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1ca97fa640 20 link_request req_data=0x55e89b47ced0 req_data->id=29, curl_handle=0x55e89b534cc0 2026-03-20T11:45:51.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d127fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d127fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-21 info.flags=0x0 2026-03-20T11:45:51.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d127fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-21 to cache LRU end 2026-03-20T11:45:51.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d127fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-21 : miss 2026-03-20T11:45:51.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.877+0000 7f1d127fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d127fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d127fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-21 info.flags=0x0 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d127fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-21 to cache LRU end 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d127fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26): adding source pipe: {s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d127fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26): adding dest pipe: {s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d127fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d127fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d127fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b95e3f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b95e3f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b95e3f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b95e3f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b47e6b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b981060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b981060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 84d963b752cdad6a29012022cec05ced0ecf8f6597345a87fdae21ea808465cf 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:84d963b752cdad6a29012022cec05ced0ecf8f6597345a87fdae21ea808465cf 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 5080a02039cbfcc83cceda8f0d30a43a33c0d0b045db84e7c4f763a59dfb1ff9 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5080a02039cbfcc83cceda8f0d30a43a33c0d0b045db84e7c4f763a59dfb1ff9 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=30, curl_handle=0x55e89b8e9f10 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1d4f412900 20 run: stack=0x55e89b47ccf0 is io blocked 2026-03-20T11:45:51.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.878+0000 7f1ca97fa640 20 link_request req_data=0x55e89b8e6010 req_data->id=30, curl_handle=0x55e89b8e9f10 2026-03-20T11:45:51.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b479f30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b479f30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b479f30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b479f30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b949e10:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 15 stack 0x55e89b482230 end 2026-03-20T11:45:51.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is done 2026-03-20T11:45:51.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 collect(): s=0x55e89b7ed970 stack=0x55e89b482230 is complete 2026-03-20T11:45:51.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 10 trimming bilogs for bucket=:dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) markers=[,00000000002.6.6,,00000000001.10.6,,00000000001.6.6,00000000002.6.6,00000000001.6.6,00000000001.12.6,00000000001.4.6,], shards=11 2026-03-20T11:45:51.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b479f30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 10 trimming bilog shard 1 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000002.6.6 2026-03-20T11:45:51.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 10 trimming bilog shard 3 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.10.6 2026-03-20T11:45:51.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 10 trimming bilog shard 5 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.6.6 2026-03-20T11:45:51.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 10 trimming bilog shard 6 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000002.6.6 2026-03-20T11:45:51.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 10 trimming bilog shard 7 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.6.6 2026-03-20T11:45:51.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 10 trimming bilog shard 8 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.12.6 2026-03-20T11:45:51.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 10 trimming bilog shard 9 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.4.6 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b943280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b9816c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 cr:s=0x55e89b951140:op=0x55e89b9589f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b954310:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 cr:s=0x55e89b95c150:op=0x55e89b95a250:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 cr:s=0x55e89b955f10:op=0x55e89b94fc90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 cr:s=0x55e89b953170:op=0x55e89b980190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b943280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.1 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is io blocked 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b9816c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.3 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 run: stack=0x55e89b95b890 is io blocked 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 cr:s=0x55e89b951140:op=0x55e89b9589f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.5 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 run: stack=0x55e89b951140 is io blocked 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b954310:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.6 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is io blocked 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 cr:s=0x55e89b95c150:op=0x55e89b95a250:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.7 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 run: stack=0x55e89b95c150 is io blocked 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 cr:s=0x55e89b955f10:op=0x55e89b94fc90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.8 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 run: stack=0x55e89b955f10 is io blocked 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 cr:s=0x55e89b953170:op=0x55e89b980190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.9 2026-03-20T11:45:51.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.908+0000 7f1d4f412900 20 run: stack=0x55e89b953170 is io blocked 2026-03-20T11:45:51.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b9816c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b9816c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 15 stack 0x55e89b95b890 end 2026-03-20T11:45:51.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 run: stack=0x55e89b95b890 is done 2026-03-20T11:45:51.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b479f30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 cr:s=0x55e89b953170:op=0x55e89b980190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 cr:s=0x55e89b953170:op=0x55e89b980190:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 15 stack 0x55e89b953170 end 2026-03-20T11:45:51.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 run: stack=0x55e89b953170 is done 2026-03-20T11:45:51.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b479f30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b943280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b943280:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 15 stack 0x55e89b482230 end 2026-03-20T11:45:51.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is done 2026-03-20T11:45:51.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b954310:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b954310:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 15 stack 0x55e89b946f40 end 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is done 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b479f30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b479f30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 cr:s=0x55e89b951140:op=0x55e89b9589f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 cr:s=0x55e89b951140:op=0x55e89b9589f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 15 stack 0x55e89b951140 end 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 run: stack=0x55e89b951140 is done 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b479f30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 cr:s=0x55e89b95c150:op=0x55e89b95a250:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 cr:s=0x55e89b95c150:op=0x55e89b95a250:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 15 stack 0x55e89b95c150 end 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 run: stack=0x55e89b95c150 is done 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b479f30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.910+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d4f412900 20 cr:s=0x55e89b955f10:op=0x55e89b94fc90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d4f412900 20 cr:s=0x55e89b955f10:op=0x55e89b94fc90:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d4f412900 15 stack 0x55e89b955f10 end 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d4f412900 20 run: stack=0x55e89b955f10 is done 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b479f30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d4f412900 20 cr:s=0x55e89b7ed970:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d4f412900 20 trim: trimmed bucket instance dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d4f412900 15 stack 0x55e89b7ed970 end 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d4f412900 20 run: stack=0x55e89b7ed970 is done 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d4f412900 4 starting trim on bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:45:51.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b948820:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b948820:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d4f412900 20 enqueued request req=0x55e89b8e7f40 2026-03-20T11:45:51.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d4f412900 20 req: 0x55e89b8e7f40 2026-03-20T11:45:51.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is io blocked 2026-03-20T11:45:51.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d10ff9640 20 dequeued request req=0x55e89b8e7f40 2026-03-20T11:45:51.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d10ff9640 20 RGWWQ: empty 2026-03-20T11:45:51.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d10ff9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 : miss 2026-03-20T11:45:51.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d10ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d10ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:51.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d10ff9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 info.flags=0x17 2026-03-20T11:45:51.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d10ff9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 to cache LRU end 2026-03-20T11:45:51.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d10ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d10ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d10ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:45:51.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d10ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-41 : miss 2026-03-20T11:45:51.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d10ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d10ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d10ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-41 info.flags=0x0 2026-03-20T11:45:51.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d10ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-41 to cache LRU end 2026-03-20T11:45:51.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d10ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-41 : miss 2026-03-20T11:45:51.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.911+0000 7f1d10ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d10ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d10ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-41 info.flags=0x0 2026-03-20T11:45:51.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d10ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-41 to cache LRU end 2026-03-20T11:45:51.909 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d10ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1): adding source pipe: {s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d10ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1): adding dest pipe: {s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d10ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d10ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d10ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b948820:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b948820:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b948820:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b948820:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b479f30:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8f4f80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8f4f80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = e5da4c31500085d91929c11c6c2ab83e9eb44fa6957786816563b6f60c928ac8 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:e5da4c31500085d91929c11c6c2ab83e9eb44fa6957786816563b6f60c928ac8 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 76851373be76bec93e5a9d346844033cac44f9d8751568797cdaa2294d15e89c 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=76851373be76bec93e5a9d346844033cac44f9d8751568797cdaa2294d15e89c 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=31, curl_handle=0x55e89b47fb60 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is io blocked 2026-03-20T11:45:51.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.912+0000 7f1ca97fa640 20 link_request req_data=0x55e89b47c820 req_data->id=31, curl_handle=0x55e89b47fb60 2026-03-20T11:45:51.910 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:51.912+0000 7fcfef22c640 -1 req 13841221806787951616 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b951430:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d4f412900 15 stack 0x55e89b97e430 end 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d4f412900 20 run: stack=0x55e89b97e430 is done 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d4f412900 20 collect(): s=0x55e89b945850 stack=0x55e89b97e430 is complete 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d4f412900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d4f412900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d4f412900 15 stack 0x55e89b945850 end 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d4f412900 20 stack->operate() returned ret=-22 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is done 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d4f412900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d4f412900 4 starting trim on bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b948820:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b948820:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d4f412900 20 enqueued request req=0x55e89b94d8e0 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d4f412900 20 req: 0x55e89b94d8e0 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is io blocked 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d117fa640 20 dequeued request req=0x55e89b94d8e0 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d117fa640 20 RGWWQ: empty 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d117fa640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 : miss 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d117fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d117fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d117fa640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 info.flags=0x17 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d117fa640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 to cache LRU end 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d117fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d117fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d117fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d117fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-49 : miss 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d117fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d117fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d117fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-49 info.flags=0x0 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d117fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-49 to cache LRU end 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d117fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-49 : miss 2026-03-20T11:45:51.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.916+0000 7f1d117fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d117fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d117fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-49 info.flags=0x0 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d117fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-49 to cache LRU end 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d117fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9): adding source pipe: {s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d117fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9): adding dest pipe: {s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d117fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d117fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d117fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b948820:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b948820:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b948820:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b948820:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b951430:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.914 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.915 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 2ffd2402ff49130fc35e65fb9ae3893bb6af22dd9384ae699bdf1ce52192b3a0 2026-03-20T11:45:51.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.915 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.915 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.915 INFO:teuthology.orchestra.run.vm02.stderr:2ffd2402ff49130fc35e65fb9ae3893bb6af22dd9384ae699bdf1ce52192b3a0 2026-03-20T11:45:51.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = af2edf0778a344a02aa83fa044f7bac85658554549dd2b7a5166e5e32604f8af 2026-03-20T11:45:51.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=af2edf0778a344a02aa83fa044f7bac85658554549dd2b7a5166e5e32604f8af 2026-03-20T11:45:51.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=32, curl_handle=0x55e89b47c780 2026-03-20T11:45:51.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1d4f412900 20 run: stack=0x55e89b97e430 is io blocked 2026-03-20T11:45:51.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.917+0000 7f1ca97fa640 20 link_request req_data=0x55e89b7ed710 req_data->id=32, curl_handle=0x55e89b47c780 2026-03-20T11:45:51.916 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b957b40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b957b40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b981060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b957b40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b981060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b957b40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b981060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b8f6690:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 15 stack 0x55e89b942940 end 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is done 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b981060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b963f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 collect(): s=0x55e89b8f22f0 stack=0x55e89b942940 is complete 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 10 trimming bilogs for bucket=:dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) markers=[,00000000001.44.6,,,,00000000001.24.6,,,00000000001.44.6,,00000000001.39.6], shards=11 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b47e6b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 15 stack 0x55e89b47ccf0 end 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 run: stack=0x55e89b47ccf0 is done 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b95b030:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 10 trimming bilog shard 1 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.44.6 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 10 trimming bilog shard 5 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.24.6 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 10 trimming bilog shard 8 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.44.6 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 10 trimming bilog shard 10 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.39.6 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 run: stack=0x55e89b8f22f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 collect(): s=0x55e89b97ec80 stack=0x55e89b47ccf0 is complete 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 10 trimming bilogs for bucket=:dipepu-21[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26]) markers=[,,,,,,,,00000000001.60.14,,], shards=11 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b954310:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 cr:s=0x55e89b953170:op=0x55e89b952280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b9587a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 cr:s=0x55e89b94e490:op=0x55e89b961b20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8f6690:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 10 trimming bilog shard 8 of :dipepu-21[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26]) at marker 00000000001.60.14 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b954310:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.1 2026-03-20T11:45:51.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is io blocked 2026-03-20T11:45:51.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 cr:s=0x55e89b953170:op=0x55e89b952280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.5 2026-03-20T11:45:51.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 run: stack=0x55e89b953170 is io blocked 2026-03-20T11:45:51.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b9587a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.8 2026-03-20T11:45:51.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 run: stack=0x55e89b95b890 is io blocked 2026-03-20T11:45:51.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 cr:s=0x55e89b94e490:op=0x55e89b961b20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.10 2026-03-20T11:45:51.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 run: stack=0x55e89b94e490 is io blocked 2026-03-20T11:45:51.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b965210:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b965210:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26.8 2026-03-20T11:45:51.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.920+0000 7f1d4f412900 20 run: stack=0x55e89b47ccf0 is io blocked 2026-03-20T11:45:51.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 cr:s=0x55e89b94e490:op=0x55e89b961b20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 cr:s=0x55e89b94e490:op=0x55e89b961b20:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 15 stack 0x55e89b94e490 end 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 run: stack=0x55e89b94e490 is done 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b95b030:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 run: stack=0x55e89b8f22f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 cr:s=0x55e89b953170:op=0x55e89b952280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 cr:s=0x55e89b953170:op=0x55e89b952280:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 15 stack 0x55e89b953170 end 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 run: stack=0x55e89b953170 is done 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b9587a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b9587a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 15 stack 0x55e89b95b890 end 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 run: stack=0x55e89b95b890 is done 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b95b030:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b95b030:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 run: stack=0x55e89b8f22f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b965210:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b965210:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 15 stack 0x55e89b47ccf0 end 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 run: stack=0x55e89b47ccf0 is done 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8f6690:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 trim: trimmed bucket instance dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 15 stack 0x55e89b97ec80 end 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is done 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 4 starting trim on bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b95a250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b95a250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 enqueued request req=0x55e89b8e1710 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 req: 0x55e89b8e1710 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is io blocked 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d07fff640 20 dequeued request req=0x55e89b8e1710 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d07fff640 20 RGWWQ: empty 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d07fff640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 : miss 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d07fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b954310:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b954310:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 15 stack 0x55e89b942940 end 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is done 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b95b030:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 cr:s=0x55e89b8f22f0:op=0x55e89b963f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 trim: trimmed bucket instance dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 15 stack 0x55e89b8f22f0 end 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 run: stack=0x55e89b8f22f0 is done 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b9625a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 4 starting trim on bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b954310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b954310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 enqueued request req=0x55e89b97df60 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 req: 0x55e89b97df60 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is io blocked 2026-03-20T11:45:51.920 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d11ffb640 20 dequeued request req=0x55e89b97df60 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d11ffb640 20 RGWWQ: empty 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d11ffb640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 : miss 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d11ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d07fff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d07fff640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 info.flags=0x17 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d07fff640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 to cache LRU end 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d07fff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d07fff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d07fff640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d07fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d07fff640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-45 : miss 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d07fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d11ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d11ffb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 info.flags=0x17 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d11ffb640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 to cache LRU end 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d11ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d11ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d11ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d11ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d11ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-33 : miss 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d11ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d07fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d07fff640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-45 info.flags=0x0 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d07fff640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-45 to cache LRU end 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d07fff640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-45 : miss 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d07fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d11ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d11ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-33 info.flags=0x0 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d11ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-33 to cache LRU end 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d11ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-33 : miss 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.922+0000 7f1d11ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d07fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d07fff640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-45 info.flags=0x0 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d07fff640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-45 to cache LRU end 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d07fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5): adding source pipe: {s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d07fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5): adding dest pipe: {s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d07fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d07fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d07fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b95a250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b95a250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b95a250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b95a250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b95b030:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d11ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d11ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-33 info.flags=0x0 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d11ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-33 to cache LRU end 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d11ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38): adding source pipe: {s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d11ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38): adding dest pipe: {s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d11ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d11ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 878974128f7a9643ea715c4be18c9058d24fe8de8321ab1dbe474d9b8b5366f0 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:878974128f7a9643ea715c4be18c9058d24fe8de8321ab1dbe474d9b8b5366f0 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d11ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 0f5be5004aae337308363cf087f699731662b102d18b122df820ced0869b1ffe 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=0f5be5004aae337308363cf087f699731662b102d18b122df820ced0869b1ffe 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=33, curl_handle=0x55e89b8e9f10 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 run: stack=0x55e89b47ccf0 is io blocked 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b954310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b954310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b954310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b954310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1ca97fa640 20 link_request req_data=0x55e89b8e6010 req_data->id=33, curl_handle=0x55e89b8e9f10 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b9625a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b9808f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b9625a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b8f6690:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b8f6690:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.923 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.923 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.923 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.923 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.923 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.923 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.923 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.923 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.923 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = b3065d034d39536ac5b131279a0de86cb6e06fa0b0ee1ba830a91ef9e6e1964f 2026-03-20T11:45:51.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.923 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.923 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.923 INFO:teuthology.orchestra.run.vm02.stderr:b3065d034d39536ac5b131279a0de86cb6e06fa0b0ee1ba830a91ef9e6e1964f 2026-03-20T11:45:51.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 9d91674d1a9e9988be1e2ed639610efdc73991410f216a129942c089e2d85bea 2026-03-20T11:45:51.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=9d91674d1a9e9988be1e2ed639610efdc73991410f216a129942c089e2d85bea 2026-03-20T11:45:51.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=34, curl_handle=0x55e89b534cc0 2026-03-20T11:45:51.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1d4f412900 20 run: stack=0x55e89b95b890 is io blocked 2026-03-20T11:45:51.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.923+0000 7f1ca97fa640 20 link_request req_data=0x55e89b47ced0 req_data->id=34, curl_handle=0x55e89b534cc0 2026-03-20T11:45:51.923 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:51.923+0000 7fd01026e640 -1 req 7912502305030932157 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:51.923 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:51.923+0000 7fd008a5f640 -1 req 9540839979506967220 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8f4f80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b8f4f80:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b479f30:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d4f412900 15 stack 0x55e89b482230 end 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is done 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d4f412900 20 collect(): s=0x55e89b946f40 stack=0x55e89b482230 is complete 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d4f412900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d4f412900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d4f412900 15 stack 0x55e89b946f40 end 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d4f412900 20 stack->operate() returned ret=-22 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is done 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d4f412900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d4f412900 4 starting trim on bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b95a250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b95a250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d4f412900 20 enqueued request req=0x55e89b8e7f40 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d4f412900 20 req: 0x55e89b8e7f40 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is io blocked 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d077fe640 20 dequeued request req=0x55e89b8e7f40 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d077fe640 20 RGWWQ: empty 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d077fe640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 : miss 2026-03-20T11:45:51.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d077fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d077fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d077fe640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 info.flags=0x17 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d077fe640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 to cache LRU end 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d077fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d077fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d077fe640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d077fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d077fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-47 : miss 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.954+0000 7f1d077fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d077fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d077fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-47 info.flags=0x0 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d077fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-47 to cache LRU end 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d077fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-47 : miss 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d077fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d077fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d077fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-47 info.flags=0x0 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d077fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-47 to cache LRU end 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d077fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7): adding source pipe: {s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d077fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7): adding dest pipe: {s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d077fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d077fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d077fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b95a250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b95a250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b95a250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b95a250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b479f30:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b981f20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b981f20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = df2f9da05a5bc0023396b2852772aa5f53d5423d1b853959966d8c5ac4e5ebaf 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:df2f9da05a5bc0023396b2852772aa5f53d5423d1b853959966d8c5ac4e5ebaf 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 827585e1cf4a0019bc69f04487d3c9f373d7f6a557e5e9f5445ffe6067f8be28 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=827585e1cf4a0019bc69f04487d3c9f373d7f6a557e5e9f5445ffe6067f8be28 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=35, curl_handle=0x55e89b47fb60 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is io blocked 2026-03-20T11:45:51.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.955+0000 7f1ca97fa640 20 link_request req_data=0x55e89b47c820 req_data->id=35, curl_handle=0x55e89b47fb60 2026-03-20T11:45:51.953 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:51.955+0000 7fd012272640 -1 req 9999091450449949946 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b951430:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d4f412900 15 stack 0x55e89b97e430 end 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d4f412900 20 run: stack=0x55e89b97e430 is done 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d4f412900 20 collect(): s=0x55e89b945850 stack=0x55e89b97e430 is complete 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d4f412900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d4f412900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d4f412900 15 stack 0x55e89b945850 end 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d4f412900 20 stack->operate() returned ret=-22 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is done 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d4f412900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d4f412900 4 starting trim on bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d4f412900 20 enqueued request req=0x55e89b7ed710 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d4f412900 20 req: 0x55e89b7ed710 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is io blocked 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d06ffd640 20 dequeued request req=0x55e89b7ed710 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d06ffd640 20 RGWWQ: empty 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d06ffd640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 : miss 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d06ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d06ffd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d06ffd640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 info.flags=0x17 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d06ffd640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 to cache LRU end 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d06ffd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d06ffd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d06ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d06ffd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-58 : miss 2026-03-20T11:45:51.955 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.958+0000 7f1d06ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d06ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d06ffd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-58 info.flags=0x0 2026-03-20T11:45:51.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d06ffd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-58 to cache LRU end 2026-03-20T11:45:51.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d06ffd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-58 : miss 2026-03-20T11:45:51.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d06ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d06ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d06ffd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-58 info.flags=0x0 2026-03-20T11:45:51.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d06ffd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-58 to cache LRU end 2026-03-20T11:45:51.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d06ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18): adding source pipe: {s={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d06ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18): adding dest pipe: {s={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d06ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d06ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d06ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:45:51.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b951430:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = a648869589ddfedb49342eddac9e3d76d2bd7b96fabb7775d76f0b8a23adb119 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:a648869589ddfedb49342eddac9e3d76d2bd7b96fabb7775d76f0b8a23adb119 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = bdc67e52b8482e16fb65403187753de15574023593a368b4716c2e44a6dffa12 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=bdc67e52b8482e16fb65403187753de15574023593a368b4716c2e44a6dffa12 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=36, curl_handle=0x55e89b47c780 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1d4f412900 20 run: stack=0x55e89b97e430 is io blocked 2026-03-20T11:45:51.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.959+0000 7f1ca97fa640 20 link_request req_data=0x55e89b8df6e0 req_data->id=36, curl_handle=0x55e89b47c780 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b95b030:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 15 stack 0x55e89b47ccf0 end 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 run: stack=0x55e89b47ccf0 is done 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b8f6690:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b8f6690:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 collect(): s=0x55e89b97ec80 stack=0x55e89b47ccf0 is complete 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 15 stack 0x55e89b97ec80 end 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 stack->operate() returned ret=-22 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is done 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b9808f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 15 stack 0x55e89b95b890 end 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 run: stack=0x55e89b95b890 is done 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b9625a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 collect(): s=0x55e89b942940 stack=0x55e89b95b890 is complete 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b9625a0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 15 stack 0x55e89b942940 end 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 stack->operate() returned ret=-22 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is done 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 4 starting trim on bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.963 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b957b40:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.963 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 4 starting trim on bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:45:51.963 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.963 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 enqueued request req=0x55e89b946940 2026-03-20T11:45:51.963 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.963 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 req: 0x55e89b946940 2026-03-20T11:45:51.963 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is io blocked 2026-03-20T11:45:51.963 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b9489d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.963 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b9489d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.963 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 enqueued request req=0x55e89b479a90 2026-03-20T11:45:51.963 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.963 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 req: 0x55e89b946940 2026-03-20T11:45:51.963 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 req: 0x55e89b479a90 2026-03-20T11:45:51.963 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is io blocked 2026-03-20T11:45:51.963 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d05ffb640 20 dequeued request req=0x55e89b946940 2026-03-20T11:45:51.963 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d05ffb640 20 RGWWQ: 2026-03-20T11:45:51.963 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d05ffb640 20 req: 0x55e89b479a90 2026-03-20T11:45:51.963 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d05ffb640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 : miss 2026-03-20T11:45:51.963 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d05ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.963 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d067fc640 20 dequeued request req=0x55e89b479a90 2026-03-20T11:45:51.963 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d067fc640 20 RGWWQ: empty 2026-03-20T11:45:51.963 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d067fc640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 : miss 2026-03-20T11:45:51.963 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d067fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d067fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d067fc640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 info.flags=0x17 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d067fc640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 to cache LRU end 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d067fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d067fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d067fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d067fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-20 : miss 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d067fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d05ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d05ffb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 info.flags=0x17 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d05ffb640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 to cache LRU end 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d05ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d05ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d05ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d05ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-30 : miss 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.965+0000 7f1d05ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d067fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d067fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-20 info.flags=0x0 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d067fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-20 to cache LRU end 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d067fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-20 : miss 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d067fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d05ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d05ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-30 info.flags=0x0 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d05ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-30 to cache LRU end 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d05ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-30 : miss 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d05ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d05ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d05ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-30 info.flags=0x0 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d05ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-30 to cache LRU end 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d05ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35): adding source pipe: {s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d05ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35): adding dest pipe: {s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d05ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d05ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d05ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d067fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d067fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-20 info.flags=0x0 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d067fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-20 to cache LRU end 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d067fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25): adding source pipe: {s={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d067fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25): adding dest pipe: {s={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d067fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b95b030:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d067fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d067fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = b97012c879ca7007a25886d59c1269a6a9f224bdbefcbef8ebf5f777e5a5f109 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:b97012c879ca7007a25886d59c1269a6a9f224bdbefcbef8ebf5f777e5a5f109 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 3fbb14810336eed1544d77492aeb95c4f1dde6e9aa45dc6ac6bbc3406430a718 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=3fbb14810336eed1544d77492aeb95c4f1dde6e9aa45dc6ac6bbc3406430a718 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=37, curl_handle=0x55e89b534cc0 2026-03-20T11:45:51.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1ca97fa640 20 link_request req_data=0x55e89b47ced0 req_data->id=37, curl_handle=0x55e89b534cc0 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 run: stack=0x55e89b95b890 is io blocked 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b9489d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b9489d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b9489d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b9489d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b957b40:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b9808c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b957b40:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b8f6690:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b8f6690:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = a79bb58e085cdf94dfddb0c7361d53729bd758f114a994d8718813fd2ce691e6 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:a79bb58e085cdf94dfddb0c7361d53729bd758f114a994d8718813fd2ce691e6 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 66502e2535b073a2dada8026b55c762b1e3bc33f2ba056509938af150053ed33 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=66502e2535b073a2dada8026b55c762b1e3bc33f2ba056509938af150053ed33 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=38, curl_handle=0x55e89b8e9f10 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1d4f412900 20 run: stack=0x55e89b47ccf0 is io blocked 2026-03-20T11:45:51.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.966+0000 7f1ca97fa640 20 link_request req_data=0x55e89b8e6010 req_data->id=38, curl_handle=0x55e89b8e9f10 2026-03-20T11:45:51.966 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:51.966+0000 7fd01ea8b640 -1 req 2565463387485954755 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b981f20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b981f20:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b479f30:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1d4f412900 15 stack 0x55e89b482230 end 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is done 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1d4f412900 20 collect(): s=0x55e89b946f40 stack=0x55e89b482230 is complete 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1d4f412900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1d4f412900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1d4f412900 15 stack 0x55e89b946f40 end 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1d4f412900 20 stack->operate() returned ret=-22 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is done 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1d4f412900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1d4f412900 4 starting trim on bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1d4f412900 20 enqueued request req=0x55e89b8e7f40 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1d4f412900 20 req: 0x55e89b8e7f40 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is io blocked 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1cfafe5640 20 dequeued request req=0x55e89b8e7f40 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1cfafe5640 20 RGWWQ: empty 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1cfafe5640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 : miss 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1cfafe5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1cfafe5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1cfafe5640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 info.flags=0x17 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1cfafe5640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 to cache LRU end 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1cfafe5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1cfafe5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1cfafe5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:45:51.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1cfafe5640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-52 : miss 2026-03-20T11:45:51.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.997+0000 7f1cfafe5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1cfafe5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1cfafe5640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-52 info.flags=0x0 2026-03-20T11:45:51.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1cfafe5640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-52 to cache LRU end 2026-03-20T11:45:51.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1cfafe5640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-52 : miss 2026-03-20T11:45:51.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1cfafe5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:51.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1cfafe5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:51.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1cfafe5640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-52 info.flags=0x0 2026-03-20T11:45:51.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1cfafe5640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-52 to cache LRU end 2026-03-20T11:45:51.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1cfafe5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12): adding source pipe: {s={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1cfafe5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12): adding dest pipe: {s={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1cfafe5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:51.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1cfafe5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:51.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1cfafe5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b479f30:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b981f20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b981f20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:51 +0000 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114551Z 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = d93172ead8272a8a7fa7a1beb3e8ee153490447e19c99d9ac45891c669ca6a4b 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114551Z 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:d93172ead8272a8a7fa7a1beb3e8ee153490447e19c99d9ac45891c669ca6a4b 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 2bd4adadcd58025d1a1d745b07498231001e27ef6608b82c65b88de88b881e66 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2bd4adadcd58025d1a1d745b07498231001e27ef6608b82c65b88de88b881e66 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114551Z 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.998+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=39, curl_handle=0x55e89b47fb60 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.999+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is io blocked 2026-03-20T11:45:51.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:51.999+0000 7f1ca97fa640 20 link_request req_data=0x55e89b47c820 req_data->id=39, curl_handle=0x55e89b47fb60 2026-03-20T11:45:51.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b951430:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 15 stack 0x55e89b97e430 end 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 20 run: stack=0x55e89b97e430 is done 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 20 collect(): s=0x55e89b945850 stack=0x55e89b97e430 is complete 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 10 trimming bilogs for bucket=:dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) markers=[,00000000001.77.6,,,,00000000001.65.6,,,00000000001.20.6,,00000000001.28.6], shards=11 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b95cc10:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 10 trimming bilog shard 1 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.77.6 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 10 trimming bilog shard 5 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.65.6 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 10 trimming bilog shard 8 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.20.6 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 10 trimming bilog shard 10 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.28.6 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b8ebda0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 20 cr:s=0x55e89b953170:op=0x55e89b94fd00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 20 cr:s=0x55e89b94e490:op=0x55e89b97ee60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 20 cr:s=0x55e89b8ef2f0:op=0x55e89b9812d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b8ebda0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.1 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 20 run: stack=0x55e89b97e430 is io blocked 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 20 cr:s=0x55e89b953170:op=0x55e89b94fd00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.5 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 20 run: stack=0x55e89b953170 is io blocked 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 20 cr:s=0x55e89b94e490:op=0x55e89b97ee60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.8 2026-03-20T11:45:51.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 20 run: stack=0x55e89b94e490 is io blocked 2026-03-20T11:45:52.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 20 cr:s=0x55e89b8ef2f0:op=0x55e89b9812d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.10 2026-03-20T11:45:52.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.002+0000 7f1d4f412900 20 run: stack=0x55e89b8ef2f0 is io blocked 2026-03-20T11:45:52.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.003+0000 7f1d4f412900 20 cr:s=0x55e89b8ef2f0:op=0x55e89b9812d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.003+0000 7f1d4f412900 20 cr:s=0x55e89b8ef2f0:op=0x55e89b9812d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.003+0000 7f1d4f412900 15 stack 0x55e89b8ef2f0 end 2026-03-20T11:45:52.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.003+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.003+0000 7f1d4f412900 20 run: stack=0x55e89b8ef2f0 is done 2026-03-20T11:45:52.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.003+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b95cc10:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.003+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.003+0000 7f1d4f412900 20 cr:s=0x55e89b953170:op=0x55e89b94fd00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.003+0000 7f1d4f412900 20 cr:s=0x55e89b953170:op=0x55e89b94fd00:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.003+0000 7f1d4f412900 15 stack 0x55e89b953170 end 2026-03-20T11:45:52.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.003+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.003+0000 7f1d4f412900 20 run: stack=0x55e89b953170 is done 2026-03-20T11:45:52.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.003+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b95cc10:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.003+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d4f412900 20 cr:s=0x55e89b94e490:op=0x55e89b97ee60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d4f412900 20 cr:s=0x55e89b94e490:op=0x55e89b97ee60:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d4f412900 15 stack 0x55e89b94e490 end 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d4f412900 20 run: stack=0x55e89b94e490 is done 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b95cc10:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b8ebda0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b8ebda0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d4f412900 15 stack 0x55e89b97e430 end 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d4f412900 20 run: stack=0x55e89b97e430 is done 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b95cc10:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d4f412900 20 trim: trimmed bucket instance dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d4f412900 15 stack 0x55e89b945850 end 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is done 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d4f412900 4 starting trim on bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d4f412900 20 enqueued request req=0x55e89b7ed710 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d4f412900 20 req: 0x55e89b7ed710 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is io blocked 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d057fa640 20 dequeued request req=0x55e89b7ed710 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d057fa640 20 RGWWQ: empty 2026-03-20T11:45:52.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d057fa640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 : miss 2026-03-20T11:45:52.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.004+0000 7f1d057fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d057fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:52.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d057fa640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 info.flags=0x17 2026-03-20T11:45:52.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d057fa640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 to cache LRU end 2026-03-20T11:45:52.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d057fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:52.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d057fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:52.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d057fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:45:52.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d057fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-42 : miss 2026-03-20T11:45:52.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d057fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d057fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:52.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d057fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-42 info.flags=0x0 2026-03-20T11:45:52.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d057fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-42 to cache LRU end 2026-03-20T11:45:52.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d057fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-42 : miss 2026-03-20T11:45:52.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d057fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d057fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:52.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d057fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-42 info.flags=0x0 2026-03-20T11:45:52.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d057fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-42 to cache LRU end 2026-03-20T11:45:52.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d057fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2): adding source pipe: {s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:52.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d057fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2): adding dest pipe: {s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:52.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d057fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:52.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d057fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:52.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d057fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:45:52.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b951430:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.005+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.006+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.006+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.006+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114552Z 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.006+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.006+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114552Z 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.006+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 4bbece155a2ec4b92c212799fcafd5f953894c6da630a03ea75d34bab1efed74 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.006+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114552Z 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:4bbece155a2ec4b92c212799fcafd5f953894c6da630a03ea75d34bab1efed74 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.006+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.006+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.006+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.006+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.006+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 0e7c25c7c6c0a5e92c8db5c58f89f62eb98ac2d1cd44829e3a7ebb869f919079 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.006+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=0e7c25c7c6c0a5e92c8db5c58f89f62eb98ac2d1cd44829e3a7ebb869f919079 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.006+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.006+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114552Z 2026-03-20T11:45:52.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.006+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:52.004 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.006+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=40, curl_handle=0x55e89b47c780 2026-03-20T11:45:52.004 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.006+0000 7f1d4f412900 20 run: stack=0x55e89b97e430 is io blocked 2026-03-20T11:45:52.004 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.006+0000 7f1ca97fa640 20 link_request req_data=0x55e89b8df6e0 req_data->id=40, curl_handle=0x55e89b47c780 2026-03-20T11:45:52.004 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:52.005+0000 7fd03cac7640 -1 req 17265943363345946659 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:52.006 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.006 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:52.006 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b95b030:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:52.006 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 15 stack 0x55e89b95b890 end 2026-03-20T11:45:52.006 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 run: stack=0x55e89b95b890 is done 2026-03-20T11:45:52.006 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b8f6690:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.006 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.006 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 collect(): s=0x55e89b97ec80 stack=0x55e89b95b890 is complete 2026-03-20T11:45:52.006 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:52.006 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:52.006 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:52.006 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:52.006 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 15 stack 0x55e89b97ec80 end 2026-03-20T11:45:52.006 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 stack->operate() returned ret=-22 2026-03-20T11:45:52.006 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is done 2026-03-20T11:45:52.006 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b8f6690:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.006 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:52.006 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:52.006 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.006 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b8f6690:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.006 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.006 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 4 starting trim on bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:45:52.006 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b8f6690:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.006 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.006 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b9808c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:52.006 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 15 stack 0x55e89b47ccf0 end 2026-03-20T11:45:52.006 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 run: stack=0x55e89b47ccf0 is done 2026-03-20T11:45:52.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 enqueued request req=0x55e89b8e1710 2026-03-20T11:45:52.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:52.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 req: 0x55e89b8e1710 2026-03-20T11:45:52.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is io blocked 2026-03-20T11:45:52.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d04ff9640 20 dequeued request req=0x55e89b8e1710 2026-03-20T11:45:52.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d04ff9640 20 RGWWQ: empty 2026-03-20T11:45:52.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d04ff9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 : miss 2026-03-20T11:45:52.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d04ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b957b40:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 collect(): s=0x55e89b942940 stack=0x55e89b47ccf0 is complete 2026-03-20T11:45:52.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:52.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 10 trimming bilogs for bucket=:dipepu-20[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25]) markers=[,,,,,,,,00000000005.42.14,,], shards=11 2026-03-20T11:45:52.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b95fc20:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 10 trimming bilog shard 8 of :dipepu-20[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25]) at marker 00000000005.42.14 2026-03-20T11:45:52.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b952280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b952280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25.8 2026-03-20T11:45:52.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d4f412900 20 run: stack=0x55e89b47ccf0 is io blocked 2026-03-20T11:45:52.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d04ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:45:52.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d04ff9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 info.flags=0x17 2026-03-20T11:45:52.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d04ff9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 to cache LRU end 2026-03-20T11:45:52.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d04ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:52.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d04ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:52.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d04ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:45:52.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d04ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-2 : miss 2026-03-20T11:45:52.007 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.009+0000 7f1d04ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d04ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d04ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-2 info.flags=0x0 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d04ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-2 to cache LRU end 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d04ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-2 : miss 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d04ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d04ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d04ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-2 info.flags=0x0 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d04ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-2 to cache LRU end 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d04ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2): adding source pipe: {s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d04ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2): adding dest pipe: {s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d04ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d04ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d04ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b9808c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b8f6690:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b8f6690:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114552Z 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114552Z 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 5ba1390d7a7490eed4032cb89c0882f3ba51ea6028268fc476c48e9278d79334 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114552Z 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:5ba1390d7a7490eed4032cb89c0882f3ba51ea6028268fc476c48e9278d79334 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 9f9dac16101def55b59aee856396a36ddc7abc107ba002e7c107120352cd7d91 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=9f9dac16101def55b59aee856396a36ddc7abc107ba002e7c107120352cd7d91 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114552Z 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=41, curl_handle=0x55e89b8e9f10 2026-03-20T11:45:52.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1ca97fa640 20 link_request req_data=0x55e89b8e6010 req_data->id=41, curl_handle=0x55e89b8e9f10 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.010+0000 7f1d4f412900 20 run: stack=0x55e89b95b890 is io blocked 2026-03-20T11:45:52.009 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:52.010+0000 7fd029aa1640 -1 req 11706466768845739129 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b952280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b952280:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d4f412900 15 stack 0x55e89b47ccf0 end 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d4f412900 20 run: stack=0x55e89b47ccf0 is done 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b95fc20:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b957b40:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d4f412900 20 trim: trimmed bucket instance dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d4f412900 15 stack 0x55e89b942940 end 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is done 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b957b40:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d4f412900 4 starting trim on bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d4f412900 20 enqueued request req=0x55e89b946940 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d4f412900 20 req: 0x55e89b946940 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is io blocked 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d047f8640 20 dequeued request req=0x55e89b946940 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d047f8640 20 RGWWQ: empty 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d047f8640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 : miss 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d047f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d047f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d047f8640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 info.flags=0x17 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d047f8640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 to cache LRU end 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d047f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d047f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d047f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d047f8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-57 : miss 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d047f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d047f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d047f8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-57 info.flags=0x0 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d047f8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-57 to cache LRU end 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d047f8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-57 : miss 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.011+0000 7f1d047f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d047f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d047f8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-57 info.flags=0x0 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d047f8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-57 to cache LRU end 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d047f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17): adding source pipe: {s={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d047f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17): adding dest pipe: {s={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d047f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d047f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d047f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b949e10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b957b40:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b95a040:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b957b40:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b95d320:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b95d320:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114552Z 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114552Z 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = a80d584e2167d9f85b6eb921afec727b0d4ebc182c3c1515333e84aea634b299 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114552Z 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:a80d584e2167d9f85b6eb921afec727b0d4ebc182c3c1515333e84aea634b299 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = aaf000bc7d7a5def1989257f46abe90749d4b3d9de91ad84d53372b5d8eb045a 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=aaf000bc7d7a5def1989257f46abe90749d4b3d9de91ad84d53372b5d8eb045a 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114552Z 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=42, curl_handle=0x55e89b534cc0 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1d4f412900 20 run: stack=0x55e89b47ccf0 is io blocked 2026-03-20T11:45:52.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.012+0000 7f1ca97fa640 20 link_request req_data=0x55e89b47ced0 req_data->id=42, curl_handle=0x55e89b534cc0 2026-03-20T11:45:52.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b981f20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b981f20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b981f20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b981f20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b479f30:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:52.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 15 stack 0x55e89b482230 end 2026-03-20T11:45:52.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is done 2026-03-20T11:45:52.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 collect(): s=0x55e89b946f40 stack=0x55e89b482230 is complete 2026-03-20T11:45:52.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:52.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 10 trimming bilogs for bucket=:dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) markers=[00000000004.19.6,00000000002.34.1,00000000004.47.6,00000000002.22.1,00000000004.31.6,00000000002.67.1,00000000002.32.1,00000000004.16.6,00000000002.26.1,00000000002.76.1,00000000002.54.1], shards=11 2026-03-20T11:45:52.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b47e6b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 10 trimming bilog shard 0 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000004.19.6 2026-03-20T11:45:52.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 10 trimming bilog shard 1 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.34.1 2026-03-20T11:45:52.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 10 trimming bilog shard 2 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000004.47.6 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 10 trimming bilog shard 3 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.22.1 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 10 trimming bilog shard 4 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000004.31.6 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 10 trimming bilog shard 5 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.67.1 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 10 trimming bilog shard 6 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.32.1 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 10 trimming bilog shard 7 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000004.16.6 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 10 trimming bilog shard 8 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.26.1 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 10 trimming bilog shard 9 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.76.1 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 10 trimming bilog shard 10 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.54.1 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b943280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b94e490:op=0x55e89b945d50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b953170:op=0x55e89b961b20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b8ef2f0:op=0x55e89b981180:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b95ed20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b8efe20:op=0x55e89b8e4d70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b943a80:op=0x55e89b982690:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b948820:op=0x55e89b982e90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b959300:op=0x55e89b9634e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b946550:op=0x55e89b963ce0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b983690:op=0x55e89b9644e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b943280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.0 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is io blocked 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b94e490:op=0x55e89b945d50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.1 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 run: stack=0x55e89b94e490 is io blocked 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b953170:op=0x55e89b961b20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.2 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 run: stack=0x55e89b953170 is io blocked 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b8ef2f0:op=0x55e89b981180:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.3 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 run: stack=0x55e89b8ef2f0 is io blocked 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b95ed20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.4 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 run: stack=0x55e89b8ea3f0 is io blocked 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b8efe20:op=0x55e89b8e4d70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.5 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 run: stack=0x55e89b8efe20 is io blocked 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b943a80:op=0x55e89b982690:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.6 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 run: stack=0x55e89b943a80 is io blocked 2026-03-20T11:45:52.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b948820:op=0x55e89b982e90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.7 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 run: stack=0x55e89b948820 is io blocked 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b959300:op=0x55e89b9634e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.8 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 run: stack=0x55e89b959300 is io blocked 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b946550:op=0x55e89b963ce0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.9 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 run: stack=0x55e89b946550 is io blocked 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 cr:s=0x55e89b983690:op=0x55e89b9644e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.041+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.10 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.042+0000 7f1d4f412900 20 run: stack=0x55e89b983690 is io blocked 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.043+0000 7f1d4f412900 20 cr:s=0x55e89b943a80:op=0x55e89b982690:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.043+0000 7f1d4f412900 20 cr:s=0x55e89b943a80:op=0x55e89b982690:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.043+0000 7f1d4f412900 15 stack 0x55e89b943a80 end 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.043+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.043+0000 7f1d4f412900 20 run: stack=0x55e89b943a80 is done 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.043+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b47e6b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.043+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.043+0000 7f1d4f412900 20 cr:s=0x55e89b8ef2f0:op=0x55e89b981180:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.043+0000 7f1d4f412900 20 cr:s=0x55e89b8ef2f0:op=0x55e89b981180:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.043+0000 7f1d4f412900 15 stack 0x55e89b8ef2f0 end 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.043+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.043+0000 7f1d4f412900 20 run: stack=0x55e89b8ef2f0 is done 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.043+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b47e6b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.043+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 cr:s=0x55e89b959300:op=0x55e89b9634e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 cr:s=0x55e89b959300:op=0x55e89b9634e0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 15 stack 0x55e89b959300 end 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 run: stack=0x55e89b959300 is done 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b47e6b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b95ed20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b95ed20:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 15 stack 0x55e89b8ea3f0 end 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 run: stack=0x55e89b8ea3f0 is done 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b47e6b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 cr:s=0x55e89b94e490:op=0x55e89b945d50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 cr:s=0x55e89b94e490:op=0x55e89b945d50:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 15 stack 0x55e89b94e490 end 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 run: stack=0x55e89b94e490 is done 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b47e6b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b943280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b943280:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 15 stack 0x55e89b482230 end 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is done 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b47e6b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 cr:s=0x55e89b953170:op=0x55e89b961b20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 cr:s=0x55e89b953170:op=0x55e89b961b20:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 15 stack 0x55e89b953170 end 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 run: stack=0x55e89b953170 is done 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b47e6b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 cr:s=0x55e89b8efe20:op=0x55e89b8e4d70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 cr:s=0x55e89b8efe20:op=0x55e89b8e4d70:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 15 stack 0x55e89b8efe20 end 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 run: stack=0x55e89b8efe20 is done 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b47e6b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 cr:s=0x55e89b948820:op=0x55e89b982e90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 cr:s=0x55e89b948820:op=0x55e89b982e90:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 15 stack 0x55e89b948820 end 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 run: stack=0x55e89b948820 is done 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b47e6b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.044+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d4f412900 20 cr:s=0x55e89b946550:op=0x55e89b963ce0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d4f412900 20 cr:s=0x55e89b946550:op=0x55e89b963ce0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d4f412900 15 stack 0x55e89b946550 end 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d4f412900 20 run: stack=0x55e89b946550 is done 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b47e6b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d4f412900 20 cr:s=0x55e89b983690:op=0x55e89b9644e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d4f412900 20 cr:s=0x55e89b983690:op=0x55e89b9644e0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d4f412900 15 stack 0x55e89b983690 end 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d4f412900 20 run: stack=0x55e89b983690 is done 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b47e6b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d4f412900 20 cr:s=0x55e89b946f40:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d4f412900 20 trim: trimmed bucket instance dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d4f412900 15 stack 0x55e89b946f40 end 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d4f412900 20 run: stack=0x55e89b946f40 is done 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d4f412900 20 cr:s=0x55e89b8efe20:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d4f412900 4 starting trim on bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d4f412900 20 cr:s=0x55e89b8efe20:op=0x55e89b943280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d4f412900 20 cr:s=0x55e89b8efe20:op=0x55e89b943280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d4f412900 20 enqueued request req=0x55e89b8e7f40 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d4f412900 20 req: 0x55e89b8e7f40 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d03ff7640 20 dequeued request req=0x55e89b8e7f40 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d03ff7640 20 RGWWQ: empty 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d03ff7640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 : miss 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d03ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d4f412900 20 run: stack=0x55e89b8efe20 is io blocked 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d03ff7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d03ff7640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 info.flags=0x17 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d03ff7640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 to cache LRU end 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d03ff7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d03ff7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d03ff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d03ff7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-29 : miss 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d03ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d03ff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d03ff7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-29 info.flags=0x0 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d03ff7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-29 to cache LRU end 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d03ff7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-29 : miss 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.045+0000 7f1d03ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d03ff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d03ff7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-29 info.flags=0x0 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d03ff7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-29 to cache LRU end 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d03ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34): adding source pipe: {s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:52.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d03ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34): adding dest pipe: {s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d03ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d03ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d03ff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 20 cr:s=0x55e89b8efe20:op=0x55e89b943280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 20 cr:s=0x55e89b8efe20:op=0x55e89b943280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 20 cr:s=0x55e89b8efe20:op=0x55e89b943280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 20 cr:s=0x55e89b8efe20:op=0x55e89b943280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 20 cr:s=0x55e89b8efe20:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 20 cr:s=0x55e89b953170:op=0x55e89b981f20:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 20 cr:s=0x55e89b8efe20:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 20 run: stack=0x55e89b8efe20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 20 cr:s=0x55e89b953170:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 20 cr:s=0x55e89b953170:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114552Z 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114552Z 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 034ccc90b3b8605c2772a08df46f77d9d112fbb2dde32da31b49de4a43daad4b 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114552Z 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:034ccc90b3b8605c2772a08df46f77d9d112fbb2dde32da31b49de4a43daad4b 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 071388df4b37a7dcf094bd201d78e18bd15a2ee3c397f2ef1d518b7f11a64733 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=071388df4b37a7dcf094bd201d78e18bd15a2ee3c397f2ef1d518b7f11a64733 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114552Z 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=43, curl_handle=0x55e89b47fb60 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1d4f412900 20 run: stack=0x55e89b953170 is io blocked 2026-03-20T11:45:52.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.046+0000 7f1ca97fa640 20 link_request req_data=0x55e89b47c820 req_data->id=43, curl_handle=0x55e89b47fb60 2026-03-20T11:45:52.044 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:52.046+0000 7fd04a2e2640 -1 req 10447243628344419755 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b951430:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d4f412900 15 stack 0x55e89b97e430 end 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d4f412900 20 run: stack=0x55e89b97e430 is done 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d4f412900 20 collect(): s=0x55e89b945850 stack=0x55e89b97e430 is complete 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d4f412900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d4f412900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d4f412900 15 stack 0x55e89b945850 end 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d4f412900 20 stack->operate() returned ret=-22 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is done 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d4f412900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d4f412900 4 starting trim on bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b943280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b943280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d4f412900 20 enqueued request req=0x55e89b7ed710 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d4f412900 20 req: 0x55e89b7ed710 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is io blocked 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d037f6640 20 dequeued request req=0x55e89b7ed710 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d037f6640 20 RGWWQ: empty 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d037f6640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 : miss 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d037f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d037f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d037f6640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 info.flags=0x17 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d037f6640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 to cache LRU end 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d037f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d037f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d037f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d037f6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-23 : miss 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.048+0000 7f1d037f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d037f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d037f6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-23 info.flags=0x0 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d037f6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-23 to cache LRU end 2026-03-20T11:45:52.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d037f6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-23 : miss 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d037f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d037f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d037f6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-23 info.flags=0x0 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d037f6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-23 to cache LRU end 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d037f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28): adding source pipe: {s={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d037f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28): adding dest pipe: {s={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d037f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d037f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d037f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b943280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b943280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b943280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b943280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b951430:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114552Z 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114552Z 2026-03-20T11:45:52.046 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:52.047 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:52.047 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 42e6569e15646d5bdadc5f76c525510dbbba2c90c5f62669e913414d49f99b3c 2026-03-20T11:45:52.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:52.047 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114552Z 2026-03-20T11:45:52.047 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:52.047 INFO:teuthology.orchestra.run.vm02.stderr:42e6569e15646d5bdadc5f76c525510dbbba2c90c5f62669e913414d49f99b3c 2026-03-20T11:45:52.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:52.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:52.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:52.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:52.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 8715f41fae213bbd5fb63578302da585f7ee9193b8ef6d0c4c2bb79599c1f2d2 2026-03-20T11:45:52.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=8715f41fae213bbd5fb63578302da585f7ee9193b8ef6d0c4c2bb79599c1f2d2 2026-03-20T11:45:52.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114552Z 2026-03-20T11:45:52.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:52.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=44, curl_handle=0x55e89b47c780 2026-03-20T11:45:52.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1d4f412900 20 run: stack=0x55e89b97e430 is io blocked 2026-03-20T11:45:52.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.049+0000 7f1ca97fa640 20 link_request req_data=0x55e89b8df6e0 req_data->id=44, curl_handle=0x55e89b47c780 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b8f6690:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b8f6690:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b9808c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d4f412900 15 stack 0x55e89b95b890 end 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d4f412900 20 run: stack=0x55e89b95b890 is done 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d4f412900 20 collect(): s=0x55e89b97ec80 stack=0x55e89b95b890 is complete 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d4f412900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d4f412900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d4f412900 15 stack 0x55e89b97ec80 end 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d4f412900 20 stack->operate() returned ret=-22 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is done 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d4f412900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d4f412900 4 starting trim on bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b945050:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b945050:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d4f412900 20 enqueued request req=0x55e89b8e1710 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d4f412900 20 req: 0x55e89b8e1710 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d02ff5640 20 dequeued request req=0x55e89b8e1710 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d02ff5640 20 RGWWQ: empty 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d02ff5640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 : miss 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d02ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is io blocked 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d02ff5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d02ff5640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 info.flags=0x17 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d02ff5640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 to cache LRU end 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d02ff5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d02ff5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d02ff5640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d02ff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d02ff5640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-34 : miss 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d02ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d02ff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d02ff5640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-34 info.flags=0x0 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d02ff5640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-34 to cache LRU end 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d02ff5640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-34 : miss 2026-03-20T11:45:52.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.052+0000 7f1d02ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d02ff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d02ff5640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-34 info.flags=0x0 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d02ff5640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-34 to cache LRU end 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d02ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39): adding source pipe: {s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d02ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39): adding dest pipe: {s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d02ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d02ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d02ff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b945050:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b945050:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b945050:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b945050:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b8f6690:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b9808c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b9808c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114552Z 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114552Z 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = fec9437abf99110b4e040f657ddba292303fa98ae1ff624a3237723be8ac7e9c 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114552Z 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:fec9437abf99110b4e040f657ddba292303fa98ae1ff624a3237723be8ac7e9c 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:52.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 78f5eeeb44c533f402142fe0f5109f58987a20dffe2e924394fb5445bd4f3b0c 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=78f5eeeb44c533f402142fe0f5109f58987a20dffe2e924394fb5445bd4f3b0c 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114552Z 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=45, curl_handle=0x55e89b8e9f10 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1d4f412900 20 run: stack=0x55e89b95b890 is io blocked 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.053+0000 7f1ca97fa640 20 link_request req_data=0x55e89b8e6010 req_data->id=45, curl_handle=0x55e89b8e9f10 2026-03-20T11:45:52.051 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:52.053+0000 7fd011270640 -1 req 2127212344183164129 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.054+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b95d320:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.054+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b95d320:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.054+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b95d320:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.054+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b95d320:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.054+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b95a040:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.054+0000 7f1d4f412900 15 stack 0x55e89b47ccf0 end 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.054+0000 7f1d4f412900 20 run: stack=0x55e89b47ccf0 is done 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.054+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b957b40:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.054+0000 7f1d4f412900 20 collect(): s=0x55e89b942940 stack=0x55e89b47ccf0 is complete 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.054+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.054+0000 7f1d4f412900 10 trimming bilogs for bucket=:dipepu-57[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17]) markers=[,00000000001.41.6,,,,00000000001.23.6,,,,,], shards=11 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.054+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b945ea0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.054+0000 7f1d4f412900 10 trimming bilog shard 1 of :dipepu-57[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17]) at marker 00000000001.41.6 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.054+0000 7f1d4f412900 10 trimming bilog shard 5 of :dipepu-57[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17]) at marker 00000000001.23.6 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.054+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.054+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b952280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.054+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b959fc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.054+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b952280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.054+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17.1 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.054+0000 7f1d4f412900 20 run: stack=0x55e89b47ccf0 is io blocked 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.054+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b959fc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.054+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17.5 2026-03-20T11:45:52.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.054+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is io blocked 2026-03-20T11:45:52.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b952280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b952280:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d4f412900 15 stack 0x55e89b47ccf0 end 2026-03-20T11:45:52.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d4f412900 20 run: stack=0x55e89b47ccf0 is done 2026-03-20T11:45:52.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b945ea0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b959fc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b959fc0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d4f412900 15 stack 0x55e89b482230 end 2026-03-20T11:45:52.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is done 2026-03-20T11:45:52.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b945ea0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b957b40:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d4f412900 20 trim: trimmed bucket instance dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d4f412900 15 stack 0x55e89b942940 end 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is done 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b8e47f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d4f412900 4 starting trim on bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b945d50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b945d50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d4f412900 20 enqueued request req=0x55e89b95c9a0 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d4f412900 20 req: 0x55e89b95c9a0 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is io blocked 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d01ff3640 20 dequeued request req=0x55e89b95c9a0 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d01ff3640 20 RGWWQ: empty 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.055+0000 7f1d01ff3640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 : miss 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.056+0000 7f1d01ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.056+0000 7f1d01ff3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.056+0000 7f1d01ff3640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 info.flags=0x17 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.056+0000 7f1d01ff3640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 to cache LRU end 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.056+0000 7f1d01ff3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.056+0000 7f1d01ff3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.056+0000 7f1d01ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.056+0000 7f1d01ff3640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-54 : miss 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.056+0000 7f1d01ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.056+0000 7f1d01ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.056+0000 7f1d01ff3640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-54 info.flags=0x0 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.056+0000 7f1d01ff3640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-54 to cache LRU end 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.056+0000 7f1d01ff3640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-54 : miss 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.056+0000 7f1d01ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d01ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d01ff3640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-54 info.flags=0x0 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d01ff3640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-54 to cache LRU end 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d01ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14): adding source pipe: {s={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:52.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d01ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14): adding dest pipe: {s={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d01ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d01ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d01ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b945d50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b945d50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b945d50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b945d50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b8e47f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b95d320:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b8e47f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b945d50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b945d50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114552Z 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114552Z 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = a01d51527f6e1b48a1a437edd9f1cb0a491f26ee3660d70a0c957587054c2260 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114552Z 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:a01d51527f6e1b48a1a437edd9f1cb0a491f26ee3660d70a0c957587054c2260 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:52.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:52.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 32ea762fa830b6395b5bb3040d0070efca9839cdbfb0b0584aa68da0fcd645e7 2026-03-20T11:45:52.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=32ea762fa830b6395b5bb3040d0070efca9839cdbfb0b0584aa68da0fcd645e7 2026-03-20T11:45:52.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114552Z 2026-03-20T11:45:52.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:52.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=46, curl_handle=0x55e89b534cc0 2026-03-20T11:45:52.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is io blocked 2026-03-20T11:45:52.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.057+0000 7f1ca97fa640 20 link_request req_data=0x55e89b47ced0 req_data->id=46, curl_handle=0x55e89b534cc0 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d4f412900 20 cr:s=0x55e89b953170:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d4f412900 20 cr:s=0x55e89b953170:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d4f412900 20 cr:s=0x55e89b953170:op=0x55e89b981f20:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d4f412900 15 stack 0x55e89b953170 end 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d4f412900 20 run: stack=0x55e89b953170 is done 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d4f412900 20 cr:s=0x55e89b8efe20:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d4f412900 20 collect(): s=0x55e89b8efe20 stack=0x55e89b953170 is complete 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d4f412900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d4f412900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d4f412900 20 cr:s=0x55e89b8efe20:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d4f412900 15 stack 0x55e89b8efe20 end 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d4f412900 20 stack->operate() returned ret=-22 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d4f412900 20 run: stack=0x55e89b8efe20 is done 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d4f412900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d4f412900 20 cr:s=0x55e89b8efe20:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d4f412900 4 starting trim on bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d4f412900 20 cr:s=0x55e89b8efe20:op=0x55e89b961b20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d4f412900 20 cr:s=0x55e89b8efe20:op=0x55e89b961b20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d4f412900 20 enqueued request req=0x55e89b8e7f40 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d4f412900 20 req: 0x55e89b8e7f40 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d4f412900 20 run: stack=0x55e89b8efe20 is io blocked 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d027f4640 20 dequeued request req=0x55e89b8e7f40 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d027f4640 20 RGWWQ: empty 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d027f4640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 : miss 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d027f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d027f4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d027f4640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 info.flags=0x17 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d027f4640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 to cache LRU end 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d027f4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d027f4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d027f4640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d027f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d027f4640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-32 : miss 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.087+0000 7f1d027f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d027f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d027f4640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-32 info.flags=0x0 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d027f4640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-32 to cache LRU end 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d027f4640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-32 : miss 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d027f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d027f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d027f4640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-32 info.flags=0x0 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d027f4640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-32 to cache LRU end 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d027f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37): adding source pipe: {s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d027f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37): adding dest pipe: {s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d027f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d027f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d027f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 20 cr:s=0x55e89b8efe20:op=0x55e89b961b20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 20 cr:s=0x55e89b8efe20:op=0x55e89b961b20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 20 cr:s=0x55e89b8efe20:op=0x55e89b961b20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 20 cr:s=0x55e89b8efe20:op=0x55e89b961b20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 20 cr:s=0x55e89b8efe20:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 20 cr:s=0x55e89b953170:op=0x55e89b981f20:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 20 cr:s=0x55e89b8efe20:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 20 run: stack=0x55e89b8efe20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 20 cr:s=0x55e89b953170:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 20 cr:s=0x55e89b953170:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114552Z 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114552Z 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 9c2a108636e8a2dc3c1a5eb5835c89636dc3d3a8a3b7a6132e84041b6ebb391b 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114552Z 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:9c2a108636e8a2dc3c1a5eb5835c89636dc3d3a8a3b7a6132e84041b6ebb391b 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 1108024b2abaf539b4b1528e0b349dbbdc58e5155e6eca263f55e9b7ce8efd37 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=1108024b2abaf539b4b1528e0b349dbbdc58e5155e6eca263f55e9b7ce8efd37 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114552Z 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=47, curl_handle=0x55e89b47fb60 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1d4f412900 20 run: stack=0x55e89b953170 is io blocked 2026-03-20T11:45:52.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.088+0000 7f1ca97fa640 20 link_request req_data=0x55e89b47c820 req_data->id=47, curl_handle=0x55e89b47fb60 2026-03-20T11:45:52.086 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:52.088+0000 7fd04f2ec640 -1 req 3719439704049765867 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:52.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.092+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.092+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.092+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.092+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.092+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b951430:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:52.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.092+0000 7f1d4f412900 15 stack 0x55e89b97e430 end 2026-03-20T11:45:52.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.092+0000 7f1d4f412900 20 run: stack=0x55e89b97e430 is done 2026-03-20T11:45:52.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.092+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.092+0000 7f1d4f412900 20 collect(): s=0x55e89b945850 stack=0x55e89b97e430 is complete 2026-03-20T11:45:52.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.092+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:52.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.092+0000 7f1d4f412900 10 trimming bilogs for bucket=:dipepu-23[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28]) markers=[,,,,,,,,00000000003.16.7,,], shards=11 2026-03-20T11:45:52.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.092+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b951430:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.092+0000 7f1d4f412900 10 trimming bilog shard 8 of :dipepu-23[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28]) at marker 00000000003.16.7 2026-03-20T11:45:52.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.092+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.092+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b9830e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.092+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b9830e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.092+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28.8 2026-03-20T11:45:52.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.092+0000 7f1d4f412900 20 run: stack=0x55e89b97e430 is io blocked 2026-03-20T11:45:52.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.093+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b9830e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.093+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b9830e0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.093+0000 7f1d4f412900 15 stack 0x55e89b97e430 end 2026-03-20T11:45:52.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.093+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.093+0000 7f1d4f412900 20 run: stack=0x55e89b97e430 is done 2026-03-20T11:45:52.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.093+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b951430:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.093+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.093+0000 7f1d4f412900 20 trim: trimmed bucket instance dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:45:52.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.093+0000 7f1d4f412900 15 stack 0x55e89b945850 end 2026-03-20T11:45:52.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.093+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is done 2026-03-20T11:45:52.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.093+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:52.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.093+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.093+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.093+0000 7f1d4f412900 4 starting trim on bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:45:52.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.093+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b943280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.093+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b943280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.093+0000 7f1d4f412900 20 enqueued request req=0x55e89b7ed710 2026-03-20T11:45:52.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.093+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:52.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.093+0000 7f1d4f412900 20 req: 0x55e89b7ed710 2026-03-20T11:45:52.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.093+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is io blocked 2026-03-20T11:45:52.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.093+0000 7f1d017f2640 20 dequeued request req=0x55e89b7ed710 2026-03-20T11:45:52.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.093+0000 7f1d017f2640 20 RGWWQ: empty 2026-03-20T11:45:52.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.093+0000 7f1d017f2640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 : miss 2026-03-20T11:45:52.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.093+0000 7f1d017f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d017f2640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:52.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d017f2640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 info.flags=0x17 2026-03-20T11:45:52.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d017f2640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 to cache LRU end 2026-03-20T11:45:52.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d017f2640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:52.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d017f2640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:52.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d017f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:45:52.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d017f2640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-19 : miss 2026-03-20T11:45:52.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d017f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d017f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:52.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d017f2640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-19 info.flags=0x0 2026-03-20T11:45:52.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d017f2640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-19 to cache LRU end 2026-03-20T11:45:52.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d017f2640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-19 : miss 2026-03-20T11:45:52.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d017f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d017f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:52.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d017f2640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-19 info.flags=0x0 2026-03-20T11:45:52.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d017f2640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-19 to cache LRU end 2026-03-20T11:45:52.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d017f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24): adding source pipe: {s={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:52.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d017f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24): adding dest pipe: {s={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:52.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d017f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:52.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d017f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:52.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d017f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b943280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b943280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b943280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b943280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b951430:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114552Z 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.094+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114552Z 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = e36f25f545a835b09736b73e45b20266550c60d0b779719498090c2bace3363d 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114552Z 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:e36f25f545a835b09736b73e45b20266550c60d0b779719498090c2bace3363d 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 19591345a48cc9efdb09807c08696b005ccc0a4993d8efe0f97ad3c713badcf2 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=19591345a48cc9efdb09807c08696b005ccc0a4993d8efe0f97ad3c713badcf2 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114552Z 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=48, curl_handle=0x55e89b47c780 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 20 run: stack=0x55e89b97e430 is io blocked 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1ca97fa640 20 link_request req_data=0x55e89b8df6e0 req_data->id=48, curl_handle=0x55e89b47c780 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b9808c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b9808c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b8f6690:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 15 stack 0x55e89b95b890 end 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 20 run: stack=0x55e89b95b890 is done 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 20 collect(): s=0x55e89b97ec80 stack=0x55e89b95b890 is complete 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 15 stack 0x55e89b97ec80 end 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 20 stack->operate() returned ret=-22 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is done 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 4 starting trim on bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b943280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b943280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 20 enqueued request req=0x55e89b8e1710 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 20 RGWWQ: 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 20 req: 0x55e89b8e1710 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d00ff1640 20 dequeued request req=0x55e89b8e1710 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d00ff1640 20 RGWWQ: empty 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d00ff1640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 : miss 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d00ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is io blocked 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d00ff1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d00ff1640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 info.flags=0x17 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d00ff1640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 to cache LRU end 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d00ff1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d00ff1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d00ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d00ff1640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-4 : miss 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d00ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d00ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d00ff1640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-4 info.flags=0x0 2026-03-20T11:45:52.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d00ff1640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-4 to cache LRU end 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d00ff1640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-4 : miss 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.095+0000 7f1d00ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d00ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d00ff1640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-4 info.flags=0x0 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d00ff1640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-4 to cache LRU end 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d00ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5): adding source pipe: {s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d00ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5): adding dest pipe: {s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d00ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d00ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d00ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b943280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b943280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b943280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b943280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b943280:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b95ec00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b95ec00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114552Z 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:52 +0000 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114552Z 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 3e9102909550ad1853f2f4cd3f4e6c4fdc05b68c289f5d9c3977b9f7e6e5936b 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114552Z 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:3e9102909550ad1853f2f4cd3f4e6c4fdc05b68c289f5d9c3977b9f7e6e5936b 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 3f6b723202cf3837486455d05b3233c488ce26ccecb9681271f8cf9bccf50219 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=3f6b723202cf3837486455d05b3233c488ce26ccecb9681271f8cf9bccf50219 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114552Z 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 20 register_request mgr=0x7ffe27cb3be0 req_data->id=49, curl_handle=0x55e89b8e9f10 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1d4f412900 20 run: stack=0x55e89b95b890 is io blocked 2026-03-20T11:45:52.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.096+0000 7f1ca97fa640 20 link_request req_data=0x55e89b8e6010 req_data->id=49, curl_handle=0x55e89b8e9f10 2026-03-20T11:45:52.095 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:52.095+0000 7fd061b11640 -1 req 17786190586780275323 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b945d50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b945d50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b945d50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b945d50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b95d320:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 15 stack 0x55e89b482230 end 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is done 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b8e47f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 collect(): s=0x55e89b942940 stack=0x55e89b482230 is complete 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 10 trimming bilogs for bucket=:dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) markers=[00000000004.46.6,00000000002.37.1,00000000004.60.6,00000000002.19.1,00000000004.18.6,00000000002.57.1,00000000002.32.1,00000000004.68.6,00000000002.43.1,00000000002.85.1,00000000002.30.1], shards=11 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b95d320:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 10 trimming bilog shard 0 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.46.6 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 10 trimming bilog shard 1 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.37.1 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 10 trimming bilog shard 2 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.60.6 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 10 trimming bilog shard 3 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.19.1 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 10 trimming bilog shard 4 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.18.6 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 10 trimming bilog shard 5 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.57.1 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 10 trimming bilog shard 6 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.32.1 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 10 trimming bilog shard 7 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.68.6 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 10 trimming bilog shard 8 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.43.1 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 10 trimming bilog shard 9 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.85.1 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 10 trimming bilog shard 10 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.30.1 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b952280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b968070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b94e490:op=0x55e89b967560:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b965d60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b8ef2f0:op=0x55e89b95f4b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b943a80:op=0x55e89b966900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b964630:op=0x55e89b981060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b97f770:op=0x55e89b96bea0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b986590:op=0x55e89b9635b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b962f50:op=0x55e89b968b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b95aca0:op=0x55e89b969350:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b952280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.0 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is io blocked 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b968070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.1 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 run: stack=0x55e89b47ccf0 is io blocked 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b94e490:op=0x55e89b967560:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.2 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 run: stack=0x55e89b94e490 is io blocked 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b965d60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.3 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 run: stack=0x55e89b8ea3f0 is io blocked 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b8ef2f0:op=0x55e89b95f4b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.4 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 run: stack=0x55e89b8ef2f0 is io blocked 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b943a80:op=0x55e89b966900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.5 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 run: stack=0x55e89b943a80 is io blocked 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b964630:op=0x55e89b981060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.6 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 run: stack=0x55e89b964630 is io blocked 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b97f770:op=0x55e89b96bea0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.7 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 run: stack=0x55e89b97f770 is io blocked 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b986590:op=0x55e89b9635b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.8 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 run: stack=0x55e89b986590 is io blocked 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b962f50:op=0x55e89b968b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.9 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 run: stack=0x55e89b962f50 is io blocked 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 cr:s=0x55e89b95aca0:op=0x55e89b969350:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.10 2026-03-20T11:45:52.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.100+0000 7f1d4f412900 20 run: stack=0x55e89b95aca0 is io blocked 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b8ef2f0:op=0x55e89b95f4b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b8ef2f0:op=0x55e89b95f4b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 15 stack 0x55e89b8ef2f0 end 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 run: stack=0x55e89b8ef2f0 is done 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b95d320:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b968070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b47ccf0:op=0x55e89b968070:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 15 stack 0x55e89b47ccf0 end 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 run: stack=0x55e89b47ccf0 is done 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b952280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b482230:op=0x55e89b952280:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 15 stack 0x55e89b482230 end 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 run: stack=0x55e89b482230 is done 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b986590:op=0x55e89b9635b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b986590:op=0x55e89b9635b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 15 stack 0x55e89b986590 end 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 run: stack=0x55e89b986590 is done 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b965d60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b8ea3f0:op=0x55e89b965d60:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 15 stack 0x55e89b8ea3f0 end 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 run: stack=0x55e89b8ea3f0 is done 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b94e490:op=0x55e89b967560:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b94e490:op=0x55e89b967560:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 15 stack 0x55e89b94e490 end 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 run: stack=0x55e89b94e490 is done 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b962f50:op=0x55e89b968b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b962f50:op=0x55e89b968b50:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 15 stack 0x55e89b962f50 end 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 run: stack=0x55e89b962f50 is done 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b943a80:op=0x55e89b966900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b943a80:op=0x55e89b966900:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 15 stack 0x55e89b943a80 end 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 run: stack=0x55e89b943a80 is done 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b95aca0:op=0x55e89b969350:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b95aca0:op=0x55e89b969350:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 15 stack 0x55e89b95aca0 end 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 run: stack=0x55e89b95aca0 is done 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b97f770:op=0x55e89b96bea0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b97f770:op=0x55e89b96bea0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 15 stack 0x55e89b97f770 end 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 run: stack=0x55e89b97f770 is done 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b964630:op=0x55e89b981060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b964630:op=0x55e89b981060:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 15 stack 0x55e89b964630 end 2026-03-20T11:45:52.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 run: stack=0x55e89b964630 is done 2026-03-20T11:45:52.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b95d320:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b95d320:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b95d320:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b95d320:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b95d320:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b95d320:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b95d320:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b95d320:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b95d320:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b95d320:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b942940:op=0x55e89b8e47f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 trim: trimmed bucket instance dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:45:52.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 15 stack 0x55e89b942940 end 2026-03-20T11:45:52.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 run: stack=0x55e89b942940 is done 2026-03-20T11:45:52.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:52.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.103+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.130+0000 7f1d4f412900 20 cr:s=0x55e89b953170:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.130+0000 7f1d4f412900 20 cr:s=0x55e89b953170:op=0x55e89b47e6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:52.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.130+0000 7f1d4f412900 20 cr:s=0x55e89b953170:op=0x55e89b981f20:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:52.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.130+0000 7f1d4f412900 15 stack 0x55e89b953170 end 2026-03-20T11:45:52.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.130+0000 7f1d4f412900 20 run: stack=0x55e89b953170 is done 2026-03-20T11:45:52.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.130+0000 7f1d4f412900 20 cr:s=0x55e89b8efe20:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.130+0000 7f1d4f412900 20 collect(): s=0x55e89b8efe20 stack=0x55e89b953170 is complete 2026-03-20T11:45:52.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.130+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:52.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.130+0000 7f1d4f412900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:52.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.130+0000 7f1d4f412900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:52.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.130+0000 7f1d4f412900 20 cr:s=0x55e89b8efe20:op=0x55e89b8e32f0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:52.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.130+0000 7f1d4f412900 15 stack 0x55e89b8efe20 end 2026-03-20T11:45:52.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.130+0000 7f1d4f412900 20 stack->operate() returned ret=-22 2026-03-20T11:45:52.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.130+0000 7f1d4f412900 20 run: stack=0x55e89b8efe20 is done 2026-03-20T11:45:52.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.130+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:52.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.130+0000 7f1d4f412900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:52.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.130+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.137+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b95ec00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.137+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b95ec00:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:52.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.137+0000 7f1d4f412900 20 cr:s=0x55e89b95b890:op=0x55e89b943280:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:52.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.137+0000 7f1d4f412900 15 stack 0x55e89b95b890 end 2026-03-20T11:45:52.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.137+0000 7f1d4f412900 20 run: stack=0x55e89b95b890 is done 2026-03-20T11:45:52.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.137+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.137+0000 7f1d4f412900 20 collect(): s=0x55e89b97ec80 stack=0x55e89b95b890 is complete 2026-03-20T11:45:52.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.137+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:52.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.137+0000 7f1d4f412900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:52.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.137+0000 7f1d4f412900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:52.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.137+0000 7f1d4f412900 20 cr:s=0x55e89b97ec80:op=0x55e89b8de410:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:52.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.137+0000 7f1d4f412900 15 stack 0x55e89b97ec80 end 2026-03-20T11:45:52.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.137+0000 7f1d4f412900 20 stack->operate() returned ret=-22 2026-03-20T11:45:52.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.137+0000 7f1d4f412900 20 run: stack=0x55e89b97ec80 is done 2026-03-20T11:45:52.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.137+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:52.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.137+0000 7f1d4f412900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:52.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.137+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.138+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.138+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.138+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.138+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b94c230:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:52.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.138+0000 7f1d4f412900 20 cr:s=0x55e89b97e430:op=0x55e89b951430:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:52.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.138+0000 7f1d4f412900 15 stack 0x55e89b97e430 end 2026-03-20T11:45:52.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.138+0000 7f1d4f412900 20 run: stack=0x55e89b97e430 is done 2026-03-20T11:45:52.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.138+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.138+0000 7f1d4f412900 20 collect(): s=0x55e89b945850 stack=0x55e89b97e430 is complete 2026-03-20T11:45:52.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.138+0000 7f1d4f412900 10 min_generation is 0 2026-03-20T11:45:52.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.138+0000 7f1d4f412900 10 trimming bilogs for bucket=:dipepu-19[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24]) markers=[,,,,,,,,00000000003.27.6,,], shards=11 2026-03-20T11:45:52.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.138+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b943280:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.138+0000 7f1d4f412900 10 trimming bilog shard 8 of :dipepu-19[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24]) at marker 00000000003.27.6 2026-03-20T11:45:52.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.138+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:52.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.138+0000 7f1d4f412900 20 cr:s=0x55e89b964630:op=0x55e89b952280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.138+0000 7f1d4f412900 20 cr:s=0x55e89b964630:op=0x55e89b952280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.138+0000 7f1d4f412900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24.8 2026-03-20T11:45:52.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.138+0000 7f1d4f412900 20 run: stack=0x55e89b964630 is io blocked 2026-03-20T11:45:52.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.140+0000 7f1d4f412900 20 cr:s=0x55e89b964630:op=0x55e89b952280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:52.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.140+0000 7f1d4f412900 20 cr:s=0x55e89b964630:op=0x55e89b952280:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:52.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.140+0000 7f1d4f412900 15 stack 0x55e89b964630 end 2026-03-20T11:45:52.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.140+0000 7f1d4f412900 20 stack->operate() returned ret=-61 2026-03-20T11:45:52.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.140+0000 7f1d4f412900 20 run: stack=0x55e89b964630 is done 2026-03-20T11:45:52.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.140+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b943280:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:52.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.140+0000 7f1d4f412900 20 cr:s=0x55e89b945850:op=0x55e89b8e2020:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:52.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.140+0000 7f1d4f412900 20 trim: trimmed bucket instance dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:45:52.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.140+0000 7f1d4f412900 15 stack 0x55e89b945850 end 2026-03-20T11:45:52.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.140+0000 7f1d4f412900 20 run: stack=0x55e89b945850 is done 2026-03-20T11:45:52.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.140+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:52.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.140+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8e07a0:27BucketTrimInstanceCollectCR: operate() returned r=-22 2026-03-20T11:45:52.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.140+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8dc970:12BucketTrimCR: operate() 2026-03-20T11:45:52.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.140+0000 7f1d4f412900 20 writing bucket trim marker=12:d5ef66d5:root::.bucket.meta.dipepu-53%3a4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13:head 2026-03-20T11:45:52.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.140+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b987a40:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:52.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.140+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b987a40:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:52.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.140+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is io blocked 2026-03-20T11:45:52.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.142+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b987a40:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:52.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.142+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b987a40:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:52.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.142+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b987a40:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:52.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.142+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b987a40:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:52.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.142+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8dc970:12BucketTrimCR: operate() 2026-03-20T11:45:52.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.142+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b965260:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:52.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.142+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b965260:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:52.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.142+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is io blocked 2026-03-20T11:45:52.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.142+0000 7f1d267fc640 20 trim: bucket trim completed 2026-03-20T11:45:52.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.143+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b965260:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:52.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.143+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b965260:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:52.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.143+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b965260:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:52.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.143+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b965260:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:52.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.143+0000 7f1d4f412900 20 cr:s=0x55e89b8c41f0:op=0x55e89b8dc970:12BucketTrimCR: operate() 2026-03-20T11:45:52.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.143+0000 7f1d4f412900 4 bucket index log processing completed in 0.745997787s 2026-03-20T11:45:52.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.143+0000 7f1d4f412900 15 stack 0x55e89b8c41f0 end 2026-03-20T11:45:52.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.143+0000 7f1d4f412900 20 run: stack=0x55e89b8c41f0 is done 2026-03-20T11:45:52.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.149+0000 7f1d4f412900 20 remove_watcher() i=4 2026-03-20T11:45:52.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.149+0000 7f1d4f412900 2 removed watcher, disabling cache 2026-03-20T11:45:52.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.150+0000 7f1d4f412900 20 remove_watcher() i=5 2026-03-20T11:45:52.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.150+0000 7f1d4f412900 20 remove_watcher() i=6 2026-03-20T11:45:52.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.150+0000 7f1d4f412900 20 remove_watcher() i=0 2026-03-20T11:45:52.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.150+0000 7f1d4f412900 20 remove_watcher() i=3 2026-03-20T11:45:52.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.150+0000 7f1d4f412900 20 remove_watcher() i=2 2026-03-20T11:45:52.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.150+0000 7f1d4f412900 20 remove_watcher() i=7 2026-03-20T11:45:52.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:52.150+0000 7f1d4f412900 20 remove_watcher() i=1 2026-03-20T11:45:57.154 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '50', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:45:57.154 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '50', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:45:57.154 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bilog autotrim --rgw-sync-log-trim-max-buckets 50 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 20 --debug-ms 0 2026-03-20T11:45:57.190 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:57.190 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:57.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.209+0000 7fdeb11af900 20 rados->read ofs=0 len=0 2026-03-20T11:45:57.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.210+0000 7fdeb11af900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:45:57.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.210+0000 7fdeb11af900 20 rados->read ofs=0 len=0 2026-03-20T11:45:57.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.211+0000 7fdeb11af900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:45:57.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.211+0000 7fdeb11af900 20 realm test-realm 6a789b89-c2b8-45f5-82cc-fb80234d0c6c 2026-03-20T11:45:57.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.211+0000 7fdeb11af900 20 rados->read ofs=0 len=0 2026-03-20T11:45:57.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.211+0000 7fdeb11af900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:45:57.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.211+0000 7fdeb11af900 20 rados->read ofs=0 len=0 2026-03-20T11:45:57.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.212+0000 7fdeb11af900 20 rados_obj.operate() r=0 bl.length=10 2026-03-20T11:45:57.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.212+0000 7fdeb11af900 20 rados->read ofs=0 len=0 2026-03-20T11:45:57.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.212+0000 7fdeb11af900 20 rados_obj.operate() r=0 bl.length=2020 2026-03-20T11:45:57.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.212+0000 7fdeb11af900 20 rados->read ofs=0 len=0 2026-03-20T11:45:57.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.212+0000 7fdeb11af900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:45:57.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.212+0000 7fdeb11af900 20 rados->read ofs=0 len=0 2026-03-20T11:45:57.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.213+0000 7fdeb11af900 20 rados_obj.operate() r=0 bl.length=1030 2026-03-20T11:45:57.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.213+0000 7fdeb11af900 20 zone a1 found 2026-03-20T11:45:57.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.213+0000 7fdeb11af900 4 Realm: test-realm (6a789b89-c2b8-45f5-82cc-fb80234d0c6c) 2026-03-20T11:45:57.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.213+0000 7fdeb11af900 4 ZoneGroup: a (a) 2026-03-20T11:45:57.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.213+0000 7fdeb11af900 4 Zone: a1 (4b634ebf-58f5-45bc-839a-cb4b425e773a) 2026-03-20T11:45:57.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.213+0000 7fdeb11af900 4 using period configuration: cb2fcdce-2c1b-429a-8601-a284ee3a2381:12 2026-03-20T11:45:57.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.213+0000 7fdeb11af900 20 period zonegroup name a 2026-03-20T11:45:57.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.213+0000 7fdeb11af900 20 using current period zonegroup a 2026-03-20T11:45:57.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.213+0000 7fdeb11af900 20 rados->read ofs=0 len=0 2026-03-20T11:45:57.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.213+0000 7fdeb11af900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.213+0000 7fdeb11af900 20 rados->read ofs=0 len=0 2026-03-20T11:45:57.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.213+0000 7fdeb11af900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.213+0000 7fdeb11af900 20 init(): pipe_map (zone=4b634ebf-58f5-45bc-839a-cb4b425e773a bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:45:57.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.213+0000 7fdeb11af900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.213+0000 7fdeb11af900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.214+0000 7fdeb11af900 20 rados->read ofs=0 len=0 2026-03-20T11:45:57.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.214+0000 7fdeb11af900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.214+0000 7fdeb11af900 20 rados->read ofs=0 len=0 2026-03-20T11:45:57.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.214+0000 7fdeb11af900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.214+0000 7fdeb11af900 20 init(): pipe_map (zone=51055576-6662-409d-8abb-ecaeefd3dd80 bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:45:57.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.214+0000 7fdeb11af900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.214+0000 7fdeb11af900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.214+0000 7fdeb11af900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.214+0000 7fdeb11af900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.214+0000 7fdeb11af900 20 started sync module instance, tier type = 2026-03-20T11:45:57.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.214+0000 7fdeb11af900 20 generating connection object for zone a2 id 51055576-6662-409d-8abb-ecaeefd3dd80 2026-03-20T11:45:57.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.214+0000 7fdeb11af900 20 started zone id=4b634ebf-58f5-45bc-839a-cb4b425e773a (name=a1) with tier type = 2026-03-20T11:45:57.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.274+0000 7fdeb11af900 20 add_watcher() i=0 2026-03-20T11:45:57.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.274+0000 7fdeb11af900 20 add_watcher() i=3 2026-03-20T11:45:57.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.300+0000 7fdeb11af900 20 add_watcher() i=5 2026-03-20T11:45:57.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.300+0000 7fdeb11af900 20 add_watcher() i=6 2026-03-20T11:45:57.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.303+0000 7fdeb11af900 20 add_watcher() i=4 2026-03-20T11:45:57.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.303+0000 7fdeb11af900 20 add_watcher() i=2 2026-03-20T11:45:57.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.303+0000 7fdeb11af900 20 add_watcher() i=1 2026-03-20T11:45:57.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.307+0000 7fdeb11af900 20 add_watcher() i=7 2026-03-20T11:45:57.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.307+0000 7fdeb11af900 2 all 8 watchers are set, enabling cache 2026-03-20T11:45:57.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.320+0000 7fdeb11af900 20 rgw_check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2026-03-20T11:45:57.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.320+0000 7fdeb11af900 20 rgw_check_secure_mon_conn(): mode 1 is insecure 2026-03-20T11:45:57.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.320+0000 7fdeb11af900 5 note: GC not initialized 2026-03-20T11:45:57.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.320+0000 7fde5dfe3640 20 reqs_thread_entry: start 2026-03-20T11:45:57.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.367+0000 7fdeb11af900 20 init_complete bucket index max shards: 11 2026-03-20T11:45:57.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.367+0000 7fdeb11af900 20 Filter name: none 2026-03-20T11:45:57.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.367+0000 7fde56ffd640 20 reqs_thread_entry: start 2026-03-20T11:45:57.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.367+0000 7fde557fa640 20 reqs_thread_entry: start 2026-03-20T11:45:57.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.369+0000 7fdeb11af900 10 rgw bucket trim manager: Watching bilog.trim 2026-03-20T11:45:57.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.369+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x5599065603d0:12BucketTrimCR: operate() 2026-03-20T11:45:57.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.369+0000 7fdeb11af900 10 fetching active bucket counters 2026-03-20T11:45:57.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.369+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906562e80:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:57.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.369+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906562e80:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:57.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.369+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is io blocked 2026-03-20T11:45:57.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.369+0000 7fde8a7fc640 20 trim: get_bucket_counters: [] 2026-03-20T11:45:57.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.369+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906562e80:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:57.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.369+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906562e80:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:57.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.369+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906562e80:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:57.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.369+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906562e80:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:57.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.369+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x5599065603d0:12BucketTrimCR: operate() 2026-03-20T11:45:57.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.369+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:57.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.369+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:57.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.369+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is io blocked 2026-03-20T11:45:57.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.370+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:57.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.370+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:57.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.370+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:57.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.370+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:57.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.370+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x5599065603d0:12BucketTrimCR: operate() 2026-03-20T11:45:57.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.370+0000 7fdeb11af900 10 listing cold buckets from marker=12:d5ef66d5:root::.bucket.meta.dipepu-53%3a4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13:head 2026-03-20T11:45:57.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.370+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:14MetadataListCR: operate() 2026-03-20T11:45:57.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.370+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:14MetadataListCR: operate() 2026-03-20T11:45:57.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.370+0000 7fdeb11af900 20 enqueued request req=0x7fde6c002c90 2026-03-20T11:45:57.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.370+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.370+0000 7fdeb11af900 20 req: 0x7fde6c002c90 2026-03-20T11:45:57.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.370+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is io blocked 2026-03-20T11:45:57.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.370+0000 7fde76ffd640 20 dequeued request req=0x7fde6c002c90 2026-03-20T11:45:57.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.370+0000 7fde76ffd640 20 RGWWQ: empty 2026-03-20T11:45:57.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.370+0000 7fde76ffd640 20 rgw async rados processor: starting metadata listing at 12:d5ef66d5:root::.bucket.meta.dipepu-53%3a4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13:head 2026-03-20T11:45:57.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.371+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:45:57.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.372+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-53 2026-03-20T11:45:57.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.373+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:45:57.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.374+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:45:57.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.375+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:45:57.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.376+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:45:57.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.377+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:45:57.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.378+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-34 2026-03-20T11:45:57.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.378+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-21 2026-03-20T11:45:57.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.378+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-14 2026-03-20T11:45:57.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.378+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-50 2026-03-20T11:45:57.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.378+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-57 2026-03-20T11:45:57.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.378+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:45:57.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.378+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-39 2026-03-20T11:45:57.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.378+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:45:57.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.379+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-37 2026-03-20T11:45:57.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.379+0000 7fde76ffd640 20 rgw async rados processor: restarting metadata listing 2026-03-20T11:45:57.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.386+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-10 2026-03-20T11:45:57.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.386+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:45:57.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.392+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:45:57.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.398+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:45:57.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.404+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:45:57.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.409+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:45:57.412 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.415+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:45:57.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.421+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-19 2026-03-20T11:45:57.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.421+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-3 2026-03-20T11:45:57.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.421+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:45:57.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.426+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:45:57.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.432+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-17 2026-03-20T11:45:57.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.432+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:45:57.435 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.438+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-38 2026-03-20T11:45:57.435 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.438+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:45:57.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.444+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-24 2026-03-20T11:45:57.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.444+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-36 2026-03-20T11:45:57.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.444+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:45:57.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.450+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:45:57.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.456+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:45:57.457 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.461+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-44 2026-03-20T11:45:57.457 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.461+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-35 2026-03-20T11:45:57.457 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.461+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:45:57.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.470+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-16 2026-03-20T11:45:57.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.470+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:45:57.471 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.475+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:45:57.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.480+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 2026-03-20T11:45:57.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.485+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:45:57.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.490+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-55 2026-03-20T11:45:57.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.490+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-13 2026-03-20T11:45:57.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.490+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-48 2026-03-20T11:45:57.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.490+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-25 2026-03-20T11:45:57.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.490+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-51 2026-03-20T11:45:57.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.490+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:45:57.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.496+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:45:57.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.500+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-46 2026-03-20T11:45:57.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.500+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:45:57.501 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.505+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:45:57.505 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.509+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:45:57.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.513+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-26 2026-03-20T11:45:57.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.513+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-20 2026-03-20T11:45:57.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.513+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:45:57.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.517+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:45:57.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.521+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-1 2026-03-20T11:45:57.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.521+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:45:57.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.525+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:45:57.525 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.528+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:45:57.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.532+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-52 2026-03-20T11:45:57.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.532+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-32 2026-03-20T11:45:57.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.532+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-42 2026-03-20T11:45:57.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.532+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:45:57.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.536+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:45:57.535 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.539+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-7 2026-03-20T11:45:57.535 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.539+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-54 2026-03-20T11:45:57.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.539+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-49 2026-03-20T11:45:57.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.539+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-47 2026-03-20T11:45:57.536 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.539+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:45:57.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.542+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-11 2026-03-20T11:45:57.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.542+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-12 2026-03-20T11:45:57.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.542+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-41 2026-03-20T11:45:57.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.542+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-23 2026-03-20T11:45:57.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.542+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-15 2026-03-20T11:45:57.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.542+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-45 2026-03-20T11:45:57.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.542+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:45:57.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.544+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:45:57.543 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.547+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-9 2026-03-20T11:45:57.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.547+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:45:57.546 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.549+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-29 2026-03-20T11:45:57.546 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.549+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:45:57.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.552+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-31 2026-03-20T11:45:57.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.552+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:45:57.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.554+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-33 2026-03-20T11:45:57.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.554+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:45:57.553 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.556+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-4 2026-03-20T11:45:57.553 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.556+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:45:57.555 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.559+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:45:57.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.561+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:45:57.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.563+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-43 2026-03-20T11:45:57.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.563+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-30 2026-03-20T11:45:57.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.563+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-18 2026-03-20T11:45:57.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.563+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-2 2026-03-20T11:45:57.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.563+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:45:57.563 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-8 2026-03-20T11:45:57.563 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fde76ffd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:45:57.563 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:14MetadataListCR: operate() 2026-03-20T11:45:57.563 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:14MetadataListCR: operate() 2026-03-20T11:45:57.563 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:14MetadataListCR: operate() 2026-03-20T11:45:57.563 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:14MetadataListCR: operate() 2026-03-20T11:45:57.563 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x5599065603d0:12BucketTrimCR: operate() 2026-03-20T11:45:57.563 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 4 collected 50 buckets for trim 2026-03-20T11:45:57.563 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.563 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.563 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 cr:s=0x55990646b350:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.563 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 4 starting trim on bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:45:57.563 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 cr:s=0x5599065a06e0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.563 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 4 starting trim on bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:45:57.563 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x559906566ed0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.563 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 4 starting trim on bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:45:57.563 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065681a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.563 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 4 starting trim on bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 cr:s=0x55990646b350:op=0x559906569470:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 cr:s=0x5599065a06e0:op=0x559906569bc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x55990656a310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x55990656aa60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 cr:s=0x55990646b350:op=0x559906569470:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 enqueued request req=0x55990656b1b0 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 req: 0x55990656b1b0 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 run: stack=0x55990646b350 is io blocked 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 cr:s=0x5599065a06e0:op=0x559906569bc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 enqueued request req=0x55990656b520 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 req: 0x55990656b1b0 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 req: 0x55990656b520 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 run: stack=0x5599065a06e0 is io blocked 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x55990656a310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 enqueued request req=0x55990656b8c0 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 req: 0x55990656b1b0 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 req: 0x55990656b520 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 req: 0x55990656b8c0 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 run: stack=0x559906565150 is io blocked 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x55990656aa60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 enqueued request req=0x55990656bce0 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 req: 0x55990656b1b0 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 req: 0x55990656b520 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 req: 0x55990656b8c0 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 req: 0x55990656bce0 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fdeb11af900 20 run: stack=0x559906564fa0 is io blocked 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fde75ffb640 20 dequeued request req=0x55990656b1b0 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fde75ffb640 20 RGWWQ: 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fde75ffb640 20 req: 0x55990656b520 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fde75ffb640 20 req: 0x55990656b8c0 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fde75ffb640 20 req: 0x55990656bce0 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fde757fa640 20 dequeued request req=0x55990656b520 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fde757fa640 20 RGWWQ: 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fde757fa640 20 req: 0x55990656b8c0 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fde757fa640 20 req: 0x55990656bce0 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fde74ff9640 20 dequeued request req=0x55990656b8c0 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fde74ff9640 20 RGWWQ: 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fde74ff9640 20 req: 0x55990656bce0 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fde767fc640 20 dequeued request req=0x55990656bce0 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fde767fc640 20 RGWWQ: empty 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fde757fa640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 : miss 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fde74ff9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 : miss 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fde767fc640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 : miss 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fde767fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fde75ffb640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 : miss 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fde75ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.565+0000 7fde74ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde757fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde74ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde75ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde75ffb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 info.flags=0x17 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde75ffb640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 to cache LRU end 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde75ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde75ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde75ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde74ff9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 info.flags=0x17 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde74ff9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 to cache LRU end 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde74ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde74ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde74ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde74ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-36 : miss 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde74ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde75ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-53 : miss 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde75ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde767fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde767fc640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 info.flags=0x17 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde767fc640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 to cache LRU end 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde767fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde767fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde767fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde767fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-22 : miss 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde767fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde757fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde757fa640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 info.flags=0x17 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde757fa640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 to cache LRU end 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde757fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde757fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde757fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde757fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-14 : miss 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde757fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde74ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde74ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-36 info.flags=0x0 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde74ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-36 to cache LRU end 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde74ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-36 : miss 2026-03-20T11:45:57.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde74ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde75ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde75ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-53 info.flags=0x0 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde75ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-53 to cache LRU end 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde75ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-53 : miss 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.566+0000 7fde75ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde767fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde767fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-22 info.flags=0x0 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde767fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-22 to cache LRU end 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde767fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-22 : miss 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde767fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde74ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde74ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-36 info.flags=0x0 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde74ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-36 to cache LRU end 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde757fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde757fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-14 info.flags=0x0 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde757fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-14 to cache LRU end 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde757fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-14 : miss 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde75ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde75ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-53 info.flags=0x0 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde75ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-53 to cache LRU end 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde75ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13): adding source pipe: {s={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde757fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde75ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13): adding dest pipe: {s={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde767fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde767fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-22 info.flags=0x0 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde767fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-22 to cache LRU end 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde767fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27): adding source pipe: {s={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde767fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27): adding dest pipe: {s={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde74ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41): adding source pipe: {s={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde74ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41): adding dest pipe: {s={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde75ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde75ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde767fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde767fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde74ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde74ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde75ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde767fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fde74ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fdeb11af900 20 cr:s=0x55990646b350:op=0x559906569470:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fdeb11af900 20 cr:s=0x55990646b350:op=0x559906569470:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fdeb11af900 20 cr:s=0x55990646b350:op=0x559906569470:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fdeb11af900 20 cr:s=0x55990646b350:op=0x559906569470:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fdeb11af900 20 cr:s=0x55990646b350:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x55990656c100:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fdeb11af900 20 cr:s=0x55990646b350:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fdeb11af900 20 run: stack=0x55990646b350 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x559906569470:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x55990656aa60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x559906569470:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 48eef3537dcc13b7f61d9e9764a9772fb849f0456296dc5da4e9511ddadb9676 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:48eef3537dcc13b7f61d9e9764a9772fb849f0456296dc5da4e9511ddadb9676 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.567+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 3b5fe2e50b7ff2a92f9ff5c84dd60725133240e2542628cbb3338ad09e91bfbb 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=3b5fe2e50b7ff2a92f9ff5c84dd60725133240e2542628cbb3338ad09e91bfbb 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=0, curl_handle=0x55990656de80 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 run: stack=0x559906563410 is io blocked 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x55990656aa60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x55990656a310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fde557fa640 20 link_request req_data=0x55990656e940 req_data->id=0, curl_handle=0x55990656de80 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x55990656aa60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x55990656a310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x55990656aa60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x55990656a310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065681a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fde757fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fde757fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-14 info.flags=0x0 2026-03-20T11:45:57.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fde757fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-14 to cache LRU end 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fde757fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19): adding source pipe: {s={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x55990656a310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065703c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065681a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 run: stack=0x559906564fa0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fde757fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19): adding dest pipe: {s={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x559906566ed0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x55990656a310:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 cr:s=0x559906571640:op=0x559906571020:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x559906566ed0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 run: stack=0x559906565150 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x55990656a310:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fde757fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fde757fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fde757fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 2db4908cacee4de2de59867df8de6821050d511040303e563defb49a5ff09859 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2db4908cacee4de2de59867df8de6821050d511040303e563defb49a5ff09859 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = e8a404772ac0130d4193f68473c0d28b31291404ce103009f77a133579971482 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=e8a404772ac0130d4193f68473c0d28b31291404ce103009f77a133579971482 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=1, curl_handle=0x559906573400 2026-03-20T11:45:57.573 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 run: stack=0x5599065709e0 is io blocked 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fde557fa640 20 link_request req_data=0x559906573b10 req_data->id=1, curl_handle=0x559906573400 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 cr:s=0x559906571640:op=0x5599065717d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 cr:s=0x5599065a06e0:op=0x559906569bc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 cr:s=0x559906571640:op=0x5599065717d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 50694ea14e03115d6aa70ba3723488a7c951c762e762027d508b61241ce7bb97 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:50694ea14e03115d6aa70ba3723488a7c951c762e762027d508b61241ce7bb97 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = de47da4cad2f434994b195496f0252ec9c84a53a26c3e47c44b85d01a6bd28f4 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=de47da4cad2f434994b195496f0252ec9c84a53a26c3e47c44b85d01a6bd28f4 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=2, curl_handle=0x559906576610 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 run: stack=0x559906571640 is io blocked 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fde557fa640 20 link_request req_data=0x559906576d60 req_data->id=2, curl_handle=0x559906576610 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 cr:s=0x5599065a06e0:op=0x559906569bc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 cr:s=0x5599065a06e0:op=0x559906569bc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 cr:s=0x5599065a06e0:op=0x559906569bc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 cr:s=0x5599065a06e0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.568+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x5599065785c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.569+0000 7fdeb11af900 20 cr:s=0x5599065a06e0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.569+0000 7fdeb11af900 20 run: stack=0x5599065a06e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.569+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x559906578d70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.569+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x559906578d70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.569+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.574 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.569+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.569+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.569+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.569+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.569+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.569+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 0a0ed49656857b82796e8157ff84f645b2beb902e73efcd03922df521da2bc84 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.569+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:0a0ed49656857b82796e8157ff84f645b2beb902e73efcd03922df521da2bc84 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.569+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.569+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.569+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.569+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.569+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 39ed2c3667f64c9033538ea46baf26e0b74103ee90da4970d02a9dcdeaaa9871 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.569+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=39ed2c3667f64c9033538ea46baf26e0b74103ee90da4970d02a9dcdeaaa9871 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.569+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.569+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.569+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.569+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=3, curl_handle=0x55990657a950 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.569+0000 7fdeb11af900 20 run: stack=0x559906578be0 is io blocked 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.569+0000 7fde557fa640 20 link_request req_data=0x55990657b060 req_data->id=3, curl_handle=0x55990657a950 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.571+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x55990656a310:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.571+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x55990656a310:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.571+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x559906578d70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.571+0000 7fdeb11af900 20 cr:s=0x559906571640:op=0x5599065717d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x559906569470:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x55990656a310:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x559906578d70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x559906571640:op=0x5599065717d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x559906569470:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x55990656a310:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x559906578d70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x559906571640:op=0x5599065717d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.575 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x559906569470:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065703c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 15 stack 0x5599065709e0 end 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 run: stack=0x5599065709e0 is done 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x559906578d70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x559906571640:op=0x5599065717d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x559906569470:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065681a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 collect(): s=0x559906564fa0 stack=0x5599065709e0 is complete 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 10 trimming bilogs for bucket=:dipepu-22[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27]) markers=[,,,,,,,,00000000006.25.9,,], shards=11 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x5599065785c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 15 stack 0x559906578be0 end 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 run: stack=0x559906578be0 is done 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x559906571640:op=0x559906571020:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 15 stack 0x559906571640 end 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 run: stack=0x559906571640 is done 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x55990656c100:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 15 stack 0x559906563410 end 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 run: stack=0x559906563410 is done 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065bcf20:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 10 trimming bilog shard 8 of :dipepu-22[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27]) at marker 00000000006.25.9 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 run: stack=0x559906564fa0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x5599065a06e0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 collect(): s=0x5599065a06e0 stack=0x559906578be0 is complete 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 10 trimming bilogs for bucket=:dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) markers=[,,,,,00000000002.32.11,,,00000000003.22.6,00000000011.52.6,], shards=11 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x559906566ed0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 collect(): s=0x559906565150 stack=0x559906571640 is complete 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 10 trimming bilogs for bucket=:dipepu-36[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41]) markers=[,,,,,,,,,,00000000001.34.6], shards=11 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x55990646b350:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 collect(): s=0x55990646b350 stack=0x559906563410 is complete 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 10 trimming bilogs for bucket=:dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) markers=[00000000004.65.1,00000000002.83.1,00000000004.51.1,00000000002.21.1,00000000004.74.1,00000000002.72.1,00000000002.84.1,00000000004.73.1,00000000002.22.1,00000000002.50.1,00000000002.35.1], shards=11 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065751b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x5599065a06e0:op=0x559906571020:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 10 trimming bilog shard 5 of :dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) at marker 00000000002.32.11 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 10 trimming bilog shard 8 of :dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) at marker 00000000003.22.6 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 10 trimming bilog shard 9 of :dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) at marker 00000000011.52.6 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 run: stack=0x5599065a06e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065785c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 10 trimming bilog shard 10 of :dipepu-36[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41]) at marker 00000000001.34.6 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 run: stack=0x559906565150 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x55990646b350:op=0x5599065703a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 10 trimming bilog shard 0 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.65.1 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 10 trimming bilog shard 1 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.83.1 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 10 trimming bilog shard 2 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.51.1 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 10 trimming bilog shard 3 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.21.1 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 10 trimming bilog shard 4 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.74.1 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 10 trimming bilog shard 5 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.72.1 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 10 trimming bilog shard 6 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.84.1 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 10 trimming bilog shard 7 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.73.1 2026-03-20T11:45:57.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 10 trimming bilog shard 8 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.22.1 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 10 trimming bilog shard 9 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.50.1 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 10 trimming bilog shard 10 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.35.1 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 run: stack=0x55990646b350 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065751b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27.8 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 run: stack=0x5599065709e0 is io blocked 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065b8970:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x559906571640:op=0x5599065ac890:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x5599065aa4c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065bc400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065a5880:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x55990656dbd0:op=0x5599065ab800:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065b5f80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x5599065bbe70:op=0x5599065b3320:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x559906571780:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x55990657a6c0:op=0x55990656c100:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x559906578d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x5599065b27e0:op=0x5599065b17e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x5599065a7d90:op=0x5599065affe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x5599065b2db0:op=0x5599065b6e70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x55990656e760:op=0x5599065b7670:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065b8970:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.572+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19.5 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.573+0000 7fdeb11af900 20 run: stack=0x559906563410 is io blocked 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.573+0000 7fdeb11af900 20 cr:s=0x559906571640:op=0x5599065ac890:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.573+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19.8 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.573+0000 7fdeb11af900 20 run: stack=0x559906571640 is io blocked 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.573+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x5599065aa4c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.573+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19.9 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.573+0000 7fdeb11af900 20 run: stack=0x559906578be0 is io blocked 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.573+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065bc400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.573+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41.10 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.573+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is io blocked 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.573+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065a5880:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.573+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.0 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.573+0000 7fdeb11af900 20 run: stack=0x5599065a70d0 is io blocked 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.573+0000 7fdeb11af900 20 cr:s=0x55990656dbd0:op=0x5599065ab800:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.573+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.1 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.573+0000 7fdeb11af900 20 run: stack=0x55990656dbd0 is io blocked 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.573+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065b5f80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.573+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.2 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.573+0000 7fdeb11af900 20 run: stack=0x5599065b6cb0 is io blocked 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.573+0000 7fdeb11af900 20 cr:s=0x5599065bbe70:op=0x5599065b3320:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.577 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.573+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.3 2026-03-20T11:45:57.578 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:57.578+0000 7fd09837e640 -1 req 9772769132937496195 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:57.578 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:57.578+0000 7fd08bb65640 -1 req 15744996121989750277 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:57.578 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:57.579+0000 7fd09637a640 -1 req 18346170363230841308 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:57.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.573+0000 7fdeb11af900 20 run: stack=0x5599065bbe70 is io blocked 2026-03-20T11:45:57.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.573+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x559906571780:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.573+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.4 2026-03-20T11:45:57.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.573+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is io blocked 2026-03-20T11:45:57.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 20 cr:s=0x55990657a6c0:op=0x55990656c100:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.5 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 20 run: stack=0x55990657a6c0 is io blocked 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x559906578d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.6 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 20 run: stack=0x5599065b41f0 is io blocked 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 20 cr:s=0x5599065b27e0:op=0x5599065b17e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.7 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 20 run: stack=0x5599065b27e0 is io blocked 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 20 cr:s=0x5599065a7d90:op=0x5599065affe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.8 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 20 run: stack=0x5599065a7d90 is io blocked 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 20 cr:s=0x5599065b2db0:op=0x5599065b6e70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.9 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 20 run: stack=0x5599065b2db0 is io blocked 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 20 cr:s=0x55990656e760:op=0x5599065b7670:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.10 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 20 run: stack=0x55990656e760 is io blocked 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065751b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065751b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 15 stack 0x5599065709e0 end 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 20 run: stack=0x5599065709e0 is done 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065bcf20:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065681a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 20 trim: trimmed bucket instance dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 15 stack 0x559906564fa0 end 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 20 run: stack=0x559906564fa0 is done 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.574+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065681a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 4 starting trim on bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065751b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065751b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 enqueued request req=0x55990656b8c0 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 req: 0x55990656b8c0 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 run: stack=0x559906564fa0 is io blocked 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fde6bfff640 20 dequeued request req=0x55990656b8c0 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fde6bfff640 20 RGWWQ: empty 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fde6bfff640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 : miss 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fde6bfff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065bc400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065bc400:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 15 stack 0x5599065ad090 end 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is done 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065785c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x5599065aa4c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x5599065aa4c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 15 stack 0x559906578be0 end 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 run: stack=0x559906578be0 is done 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065b8970:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065b8970:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.582 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 15 stack 0x559906563410 end 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 run: stack=0x559906563410 is done 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x559906566ed0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 trim: trimmed bucket instance dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 15 stack 0x559906565150 end 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 run: stack=0x559906565150 is done 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 cr:s=0x5599065a06e0:op=0x559906571020:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 cr:s=0x5599065a06e0:op=0x559906571020:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 run: stack=0x5599065a06e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x559906566ed0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 4 starting trim on bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065aa4c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065aa4c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 enqueued request req=0x55990656e3b0 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 req: 0x55990656e3b0 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fdeb11af900 20 run: stack=0x559906565150 is io blocked 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fde6b7fe640 20 dequeued request req=0x55990656e3b0 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fde6b7fe640 20 RGWWQ: empty 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.575+0000 7fde6b7fe640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 : miss 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fde6b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fde6bfff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fde6bfff640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 info.flags=0x17 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fde6bfff640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 to cache LRU end 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fde6bfff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fde6bfff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fde6bfff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fde6bfff640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-25 : miss 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fde6bfff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065b5f80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065b5f80:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 15 stack 0x5599065b6cb0 end 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 run: stack=0x5599065b6cb0 is done 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 cr:s=0x55990646b350:op=0x5599065703a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 run: stack=0x55990646b350 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 cr:s=0x55990656e760:op=0x5599065b7670:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 cr:s=0x55990656e760:op=0x5599065b7670:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 15 stack 0x55990656e760 end 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 run: stack=0x55990656e760 is done 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065a5880:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065a5880:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 15 stack 0x5599065a70d0 end 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 run: stack=0x5599065a70d0 is done 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 cr:s=0x55990657a6c0:op=0x55990656c100:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 cr:s=0x55990657a6c0:op=0x55990656c100:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 15 stack 0x55990657a6c0 end 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 run: stack=0x55990657a6c0 is done 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 cr:s=0x55990656dbd0:op=0x5599065ab800:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 cr:s=0x55990656dbd0:op=0x5599065ab800:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 15 stack 0x55990656dbd0 end 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 run: stack=0x55990656dbd0 is done 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 cr:s=0x5599065bbe70:op=0x5599065b3320:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.583 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 cr:s=0x5599065bbe70:op=0x5599065b3320:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 15 stack 0x5599065bbe70 end 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 run: stack=0x5599065bbe70 is done 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 cr:s=0x55990646b350:op=0x5599065703a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 cr:s=0x55990646b350:op=0x5599065703a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 cr:s=0x55990646b350:op=0x5599065703a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 cr:s=0x55990646b350:op=0x5599065703a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 cr:s=0x55990646b350:op=0x5599065703a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fdeb11af900 20 run: stack=0x55990646b350 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fde6b7fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fde6b7fe640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 info.flags=0x17 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fde6b7fe640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 to cache LRU end 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fde6b7fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fde6b7fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fde6b7fe640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fde6b7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fde6b7fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-35 : miss 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.576+0000 7fde6b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fde6b7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fde6b7fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-35 info.flags=0x0 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fde6b7fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-35 to cache LRU end 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fde6b7fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-35 : miss 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fde6b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fdeb11af900 20 cr:s=0x5599065b27e0:op=0x5599065b17e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fdeb11af900 20 cr:s=0x5599065b27e0:op=0x5599065b17e0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fdeb11af900 15 stack 0x5599065b27e0 end 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fdeb11af900 20 run: stack=0x5599065b27e0 is done 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fdeb11af900 20 cr:s=0x55990646b350:op=0x5599065703a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fdeb11af900 20 run: stack=0x55990646b350 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fdeb11af900 20 cr:s=0x559906571640:op=0x5599065ac890:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fdeb11af900 20 cr:s=0x559906571640:op=0x5599065ac890:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fdeb11af900 15 stack 0x559906571640 end 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fdeb11af900 20 run: stack=0x559906571640 is done 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fdeb11af900 20 cr:s=0x5599065a06e0:op=0x559906571020:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fdeb11af900 20 cr:s=0x5599065a06e0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fdeb11af900 20 trim: trimmed bucket instance dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fdeb11af900 15 stack 0x5599065a06e0 end 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fdeb11af900 20 run: stack=0x5599065a06e0 is done 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fdeb11af900 20 cr:s=0x55990656dbd0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fdeb11af900 4 starting trim on bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fdeb11af900 20 cr:s=0x55990656dbd0:op=0x5599065b8970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fdeb11af900 20 cr:s=0x55990656dbd0:op=0x5599065b8970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fde6b7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fde6b7fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-35 info.flags=0x0 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fde6b7fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-35 to cache LRU end 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fde6b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40): adding source pipe: {s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fde6b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40): adding dest pipe: {s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fde6b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fdeb11af900 20 enqueued request req=0x559906563770 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fde6b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fde6b7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fdeb11af900 20 req: 0x559906563770 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.577+0000 7fdeb11af900 20 run: stack=0x55990656dbd0 is io blocked 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065aa4c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x559906578d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6affd640 20 dequeued request req=0x559906563770 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6affd640 20 RGWWQ: empty 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6affd640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 : miss 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6affd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x559906578d20:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 15 stack 0x5599065b41f0 end 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 run: stack=0x5599065b41f0 is done 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065aa4c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x55990646b350:op=0x5599065703a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 run: stack=0x55990646b350 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x5599065b2db0:op=0x5599065b6e70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x5599065b2db0:op=0x5599065b6e70:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 15 stack 0x5599065b2db0 end 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 run: stack=0x5599065b2db0 is done 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065aa4c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x55990646b350:op=0x5599065703a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 run: stack=0x55990646b350 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065aa4c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x559906566ed0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x559906569470:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x559906566ed0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 run: stack=0x559906565150 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x5599065aafd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x5599065aafd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6bfff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6bfff640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-25 info.flags=0x0 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6bfff640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-25 to cache LRU end 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6bfff640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-25 : miss 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6bfff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 65f30b313b747c6f3dc4227cd962fc42ef6d776790f9bc4a72919b661ba83489 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:65f30b313b747c6f3dc4227cd962fc42ef6d776790f9bc4a72919b661ba83489 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6affd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.585 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6affd640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 info.flags=0x17 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6affd640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 to cache LRU end 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6affd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6affd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6affd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6affd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-26 : miss 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6affd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 95d958617b8a6cf5029676497515f64fedd0f9315074bdc7d0266fde59da5fd2 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=95d958617b8a6cf5029676497515f64fedd0f9315074bdc7d0266fde59da5fd2 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=4, curl_handle=0x55990656de80 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 run: stack=0x5599065b41f0 is io blocked 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x559906571780:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde557fa640 20 link_request req_data=0x55990656e940 req_data->id=4, curl_handle=0x55990656de80 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x559906571780:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 15 stack 0x5599065b44e0 end 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6bfff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6bfff640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-25 info.flags=0x0 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6bfff640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-25 to cache LRU end 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30): adding source pipe: {s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30): adding dest pipe: {s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is done 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x55990646b350:op=0x5599065703a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 run: stack=0x55990646b350 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6bfff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065751b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065751b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6affd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6affd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-26 info.flags=0x0 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6affd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-26 to cache LRU end 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6affd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-26 : miss 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fde6affd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065751b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065751b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065681a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x5599065a7d90:op=0x5599065affe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x5599065a7d90:op=0x5599065affe0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 15 stack 0x5599065a7d90 end 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 run: stack=0x5599065a7d90 is done 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065bcee0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065681a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 run: stack=0x559906564fa0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x55990646b350:op=0x5599065703a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x559906570fb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x55990646b350:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 trim: trimmed bucket instance dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 15 stack 0x55990646b350 end 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 run: stack=0x55990646b350 is done 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.578+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x559906570fb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fde6affd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fde6affd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-26 info.flags=0x0 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fde6affd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-26 to cache LRU end 2026-03-20T11:45:57.586 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fde6affd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31): adding source pipe: {s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fde6affd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31): adding dest pipe: {s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fde6affd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fde6affd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fde6affd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 18f0e13369d43a65da64c965c01352156a2bb3e13ed6c4dbf9606102b4900f45 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:18f0e13369d43a65da64c965c01352156a2bb3e13ed6c4dbf9606102b4900f45 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 11039bbf55efcdff4c2b84d13be9ff38f6f41aef45f3fc38c8ed5a1dc8b64114 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=11039bbf55efcdff4c2b84d13be9ff38f6f41aef45f3fc38c8ed5a1dc8b64114 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=5, curl_handle=0x559906576610 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is io blocked 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fde557fa640 20 link_request req_data=0x559906576d60 req_data->id=5, curl_handle=0x559906576610 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 cr:s=0x55990656dbd0:op=0x5599065b8970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 cr:s=0x5599065a7d90:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 4 starting trim on bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 cr:s=0x55990656dbd0:op=0x5599065b8970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 cr:s=0x5599065a7d90:op=0x5599065751b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 cr:s=0x55990656dbd0:op=0x5599065b8970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 cr:s=0x5599065a7d90:op=0x5599065751b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 enqueued request req=0x559906563140 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 req: 0x559906563140 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 run: stack=0x5599065a7d90 is io blocked 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 cr:s=0x55990656dbd0:op=0x5599065b8970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 cr:s=0x55990656dbd0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065785c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fde6a7fc640 20 dequeued request req=0x559906563140 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fde6a7fc640 20 RGWWQ: empty 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fde6a7fc640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 : miss 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fde6a7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.587 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 cr:s=0x55990656dbd0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 run: stack=0x55990656dbd0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = d943d5654424c62006954005b5b22e5fd805e803c17cea7c81fbce91f35daa66 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:d943d5654424c62006954005b5b22e5fd805e803c17cea7c81fbce91f35daa66 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 16538f18613ef9a403ebcb520644d2f806df5328b2558024d11badceb9c53aca 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=16538f18613ef9a403ebcb520644d2f806df5328b2558024d11badceb9c53aca 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=6, curl_handle=0x55990657a950 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fdeb11af900 20 run: stack=0x5599065a70d0 is io blocked 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.579+0000 7fde557fa640 20 link_request req_data=0x55990657b060 req_data->id=6, curl_handle=0x55990657a950 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fde6a7fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fde6a7fc640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 info.flags=0x17 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fde6a7fc640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 to cache LRU end 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fde6a7fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fde6a7fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fde6a7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fde6a7fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-51 : miss 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fde6a7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fde6a7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fde6a7fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-51 info.flags=0x0 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fde6a7fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-51 to cache LRU end 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fde6a7fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-51 : miss 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fde6a7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fde6a7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fde6a7fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-51 info.flags=0x0 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fde6a7fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-51 to cache LRU end 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fde6a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11): adding source pipe: {s={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fde6a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11): adding dest pipe: {s={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fde6a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fde6a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fde6a7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fdeb11af900 20 cr:s=0x5599065a7d90:op=0x5599065751b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fdeb11af900 20 cr:s=0x5599065a7d90:op=0x5599065751b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fdeb11af900 20 cr:s=0x5599065a7d90:op=0x5599065751b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fdeb11af900 20 cr:s=0x5599065a7d90:op=0x5599065751b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fdeb11af900 20 cr:s=0x5599065a7d90:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065b4ff0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fdeb11af900 20 cr:s=0x5599065a7d90:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fdeb11af900 20 run: stack=0x5599065a7d90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065751b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065751b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.580+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.581+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.581+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.581+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.581+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.581+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.581+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 55877701b3e7c6102a2c1272a47f332cc05f1e99de4be73ce40f89c0e63ca387 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.581+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:55877701b3e7c6102a2c1272a47f332cc05f1e99de4be73ce40f89c0e63ca387 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.581+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.581+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.581+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.581+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.581+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = e9ab905df7d9f2a0d1bcd815f98d15c9929958904bbfe29b6528d54b82b031d9 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.581+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=e9ab905df7d9f2a0d1bcd815f98d15c9929958904bbfe29b6528d54b82b031d9 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.581+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.581+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.581+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.581+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=7, curl_handle=0x559906573400 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.581+0000 7fdeb11af900 20 run: stack=0x5599065b6cb0 is io blocked 2026-03-20T11:45:57.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.581+0000 7fde557fa640 20 link_request req_data=0x559906573b10 req_data->id=7, curl_handle=0x559906573400 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x5599065aafd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x5599065aafd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x559906569470:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 15 stack 0x5599065b41f0 end 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 run: stack=0x5599065b41f0 is done 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x559906570fb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x559906570fb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x559906566ed0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 collect(): s=0x559906565150 stack=0x5599065b41f0 is complete 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x559906566ed0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 15 stack 0x559906565150 end 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 stack->operate() returned ret=-22 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 run: stack=0x559906565150 is done 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065bcee0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 15 stack 0x5599065b44e0 end 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is done 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065681a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 collect(): s=0x559906564fa0 stack=0x5599065b44e0 is complete 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065681a0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 15 stack 0x559906564fa0 end 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 stack->operate() returned ret=-22 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 run: stack=0x559906564fa0 is done 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x559906566ed0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 4 starting trim on bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:45:57.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065a5880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065b1290:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 4 starting trim on bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065a5880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 enqueued request req=0x5599065ad7d0 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 req: 0x5599065ad7d0 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 run: stack=0x559906565150 is io blocked 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065ac890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065ac890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 enqueued request req=0x5599065bcc00 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 req: 0x5599065ad7d0 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 req: 0x5599065bcc00 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fdeb11af900 20 run: stack=0x559906564fa0 is io blocked 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fde69ffb640 20 dequeued request req=0x5599065ad7d0 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fde69ffb640 20 RGWWQ: 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fde69ffb640 20 req: 0x5599065bcc00 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fde69ffb640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 : miss 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fde697fa640 20 dequeued request req=0x5599065bcc00 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fde697fa640 20 RGWWQ: empty 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fde69ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fde697fa640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 : miss 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.620+0000 7fde697fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065785c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 15 stack 0x5599065a70d0 end 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 20 run: stack=0x5599065a70d0 is done 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 20 cr:s=0x55990656dbd0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 20 collect(): s=0x55990656dbd0 stack=0x5599065a70d0 is complete 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde69ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde69ffb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 info.flags=0x17 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde69ffb640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 to cache LRU end 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde69ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde69ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde69ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde69ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-55 : miss 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde69ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde697fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde697fa640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 info.flags=0x17 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde697fa640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 to cache LRU end 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde697fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde697fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde697fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde697fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-28 : miss 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde697fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 20 cr:s=0x55990656dbd0:op=0x559906565c00:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 15 stack 0x55990656dbd0 end 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 20 stack->operate() returned ret=-22 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 20 run: stack=0x55990656dbd0 is done 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 4 starting trim on bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:45:57.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065bc400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065bc400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 20 enqueued request req=0x559906563770 2026-03-20T11:45:57.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 20 req: 0x559906563770 2026-03-20T11:45:57.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 20 run: stack=0x5599065a70d0 is io blocked 2026-03-20T11:45:57.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde69ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde69ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-55 info.flags=0x0 2026-03-20T11:45:57.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde69ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-55 to cache LRU end 2026-03-20T11:45:57.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde69ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-55 : miss 2026-03-20T11:45:57.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde69ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde68ff9640 20 dequeued request req=0x559906563770 2026-03-20T11:45:57.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde68ff9640 20 RGWWQ: empty 2026-03-20T11:45:57.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde68ff9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 : miss 2026-03-20T11:45:57.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde68ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde697fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde697fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-28 info.flags=0x0 2026-03-20T11:45:57.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde697fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-28 to cache LRU end 2026-03-20T11:45:57.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde697fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-28 : miss 2026-03-20T11:45:57.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde697fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.620 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:57.622+0000 7fd0b63ba640 -1 req 17529319353761878898 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde69ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde69ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-55 info.flags=0x0 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde69ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-55 to cache LRU end 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde69ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding source pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde69ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding dest pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde68ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde68ff9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 info.flags=0x17 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde68ff9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 to cache LRU end 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde68ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde68ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde68ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde69ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde69ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde68ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-31 : miss 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde68ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde69ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065a5880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065a5880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065a5880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065a5880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x559906566ed0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde697fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde697fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-28 info.flags=0x0 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.621+0000 7fde697fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-28 to cache LRU end 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fde697fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding source pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fde697fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding dest pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fde697fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fde697fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fde697fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065785c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x559906566ed0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 run: stack=0x559906565150 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065ac890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065ac890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fde68ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fde68ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-31 info.flags=0x0 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fde68ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-31 to cache LRU end 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fde68ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-31 : miss 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fde68ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = e9bffbdc1c8d3882f68ad246a8831966f5404d1155bb307c300bf99d66ed631d 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:e9bffbdc1c8d3882f68ad246a8831966f5404d1155bb307c300bf99d66ed631d 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 2824d7380c9b01376015df5fdda24142f9872bd74f91e02cbfd07e5509260741 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2824d7380c9b01376015df5fdda24142f9872bd74f91e02cbfd07e5509260741 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fde68ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fde68ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-31 info.flags=0x0 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fde68ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-31 to cache LRU end 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fde68ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding source pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fde68ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding dest pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fde68ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fde68ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fde68ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=8, curl_handle=0x55990657a950 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is io blocked 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fde557fa640 20 link_request req_data=0x55990657b060 req_data->id=8, curl_handle=0x55990657a950 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065ac890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065bc400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065ac890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065bc400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065b1290:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065bc400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x559906570fb0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065b1290:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 run: stack=0x559906564fa0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065bc400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x5599065703a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x5599065703a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 065fee0dc8c4b7143a457460d09880a42486e116630afed81d777a82e43e5d31 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:065fee0dc8c4b7143a457460d09880a42486e116630afed81d777a82e43e5d31 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = e8af26f4cdb4fefa66fcdc630216182f8ced1c80692adb97fa2261407669a528 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=e8af26f4cdb4fefa66fcdc630216182f8ced1c80692adb97fa2261407669a528 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=9, curl_handle=0x559906576610 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 run: stack=0x5599065b41f0 is io blocked 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x5599065734a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fde557fa640 20 link_request req_data=0x559906576d60 req_data->id=9, curl_handle=0x559906576610 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 run: stack=0x5599065a70d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x5599065aafd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x5599065aafd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.622+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.623+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.623+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.623+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = e026d3e8cfc17e98725bcef489b879c66fa847c5769c2e7b4de7b494eff3a216 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.623+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.625 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:e026d3e8cfc17e98725bcef489b879c66fa847c5769c2e7b4de7b494eff3a216 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.623+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.623+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.623+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.623+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.623+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = bfcedaa5893978eef958da695902f7e3510f045fa56dd283fe91b316e7705707 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.623+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=bfcedaa5893978eef958da695902f7e3510f045fa56dd283fe91b316e7705707 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.623+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.623+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.623+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.623+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=10, curl_handle=0x55990656de80 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.623+0000 7fdeb11af900 20 run: stack=0x559906578be0 is io blocked 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.623+0000 7fde557fa640 20 link_request req_data=0x55990656e940 req_data->id=10, curl_handle=0x55990656de80 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065751b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065751b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065751b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065751b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065b4ff0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 15 stack 0x5599065b6cb0 end 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 run: stack=0x5599065b6cb0 is done 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 cr:s=0x5599065a7d90:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 collect(): s=0x5599065a7d90 stack=0x5599065b6cb0 is complete 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 10 trimming bilogs for bucket=:dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) markers=[00000000004.41.6,00000000002.16.1,00000000004.28.6,00000000002.34.1,00000000004.45.6,00000000002.40.1,00000000002.39.1,00000000004.52.6,00000000002.73.1,00000000002.41.1,00000000002.74.1], shards=11 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 cr:s=0x5599065a7d90:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 10 trimming bilog shard 0 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000004.41.6 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 10 trimming bilog shard 1 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.16.1 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 10 trimming bilog shard 2 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000004.28.6 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 10 trimming bilog shard 3 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.34.1 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 10 trimming bilog shard 4 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000004.45.6 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 10 trimming bilog shard 5 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.40.1 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 10 trimming bilog shard 6 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.39.1 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 10 trimming bilog shard 7 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000004.52.6 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 10 trimming bilog shard 8 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.73.1 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 10 trimming bilog shard 9 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.41.1 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 10 trimming bilog shard 10 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.74.1 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 run: stack=0x5599065a7d90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065a5880:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065bc400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065ac6d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065bf8f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065b9700:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x559906568ee0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065681a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 cr:s=0x5599065bac10:op=0x5599065b5f80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 cr:s=0x5599065ba9b0:op=0x5599065a9fb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 cr:s=0x5599065c45f0:op=0x5599065bec00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 cr:s=0x559906471fe0:op=0x5599065ab800:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065a5880:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.0 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 run: stack=0x5599065b6cb0 is io blocked 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065bc400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.1 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 run: stack=0x559906563410 is io blocked 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065ac6d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.624+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.2 2026-03-20T11:45:57.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.625+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is io blocked 2026-03-20T11:45:57.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.625+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065bf8f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.625+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.3 2026-03-20T11:45:57.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.625+0000 7fdeb11af900 20 run: stack=0x5599065709e0 is io blocked 2026-03-20T11:45:57.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.625+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065b9700:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.625+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.4 2026-03-20T11:45:57.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.625+0000 7fdeb11af900 20 run: stack=0x5599065b43a0 is io blocked 2026-03-20T11:45:57.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.625+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x559906568ee0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.625+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.5 2026-03-20T11:45:57.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.625+0000 7fdeb11af900 20 run: stack=0x5599065a78d0 is io blocked 2026-03-20T11:45:57.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.625+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065681a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.625+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.6 2026-03-20T11:45:57.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.625+0000 7fdeb11af900 20 run: stack=0x55990656b1b0 is io blocked 2026-03-20T11:45:57.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.625+0000 7fdeb11af900 20 cr:s=0x5599065bac10:op=0x5599065b5f80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.625+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.7 2026-03-20T11:45:57.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.626+0000 7fdeb11af900 20 run: stack=0x5599065bac10 is io blocked 2026-03-20T11:45:57.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.626+0000 7fdeb11af900 20 cr:s=0x5599065ba9b0:op=0x5599065a9fb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.626+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.8 2026-03-20T11:45:57.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.626+0000 7fdeb11af900 20 run: stack=0x5599065ba9b0 is io blocked 2026-03-20T11:45:57.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.626+0000 7fdeb11af900 20 cr:s=0x5599065c45f0:op=0x5599065bec00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.626+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.9 2026-03-20T11:45:57.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.626+0000 7fdeb11af900 20 run: stack=0x5599065c45f0 is io blocked 2026-03-20T11:45:57.627 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:57.623+0000 7fd081b51640 -1 req 7962736429470540167 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:57.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 cr:s=0x559906471fe0:op=0x5599065ab800:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.10 2026-03-20T11:45:57.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 run: stack=0x559906471fe0 is io blocked 2026-03-20T11:45:57.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065a5880:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065a5880:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 15 stack 0x5599065b6cb0 end 2026-03-20T11:45:57.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 run: stack=0x5599065b6cb0 is done 2026-03-20T11:45:57.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065ac6d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065ac6d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 15 stack 0x5599065ad090 end 2026-03-20T11:45:57.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is done 2026-03-20T11:45:57.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065bf8f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065bf8f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 15 stack 0x5599065709e0 end 2026-03-20T11:45:57.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 run: stack=0x5599065709e0 is done 2026-03-20T11:45:57.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065b9700:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065b9700:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 15 stack 0x5599065b43a0 end 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 run: stack=0x5599065b43a0 is done 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 cr:s=0x5599065bac10:op=0x5599065b5f80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 cr:s=0x5599065bac10:op=0x5599065b5f80:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 15 stack 0x5599065bac10 end 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 run: stack=0x5599065bac10 is done 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x559906568ee0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x559906568ee0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 15 stack 0x5599065a78d0 end 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 run: stack=0x5599065a78d0 is done 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 cr:s=0x5599065c45f0:op=0x5599065bec00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 cr:s=0x5599065c45f0:op=0x5599065bec00:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 15 stack 0x5599065c45f0 end 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 run: stack=0x5599065c45f0 is done 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 cr:s=0x5599065a7d90:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065bc400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065bc400:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 15 stack 0x559906563410 end 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 run: stack=0x559906563410 is done 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 cr:s=0x5599065a7d90:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 cr:s=0x5599065ba9b0:op=0x5599065a9fb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 cr:s=0x5599065ba9b0:op=0x5599065a9fb0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 15 stack 0x5599065ba9b0 end 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.627+0000 7fdeb11af900 20 run: stack=0x5599065ba9b0 is done 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 cr:s=0x5599065a7d90:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065681a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065681a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 15 stack 0x55990656b1b0 end 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 run: stack=0x55990656b1b0 is done 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 cr:s=0x5599065a7d90:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 cr:s=0x5599065a7d90:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 cr:s=0x5599065a7d90:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 cr:s=0x5599065a7d90:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 cr:s=0x5599065a7d90:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 cr:s=0x5599065a7d90:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 cr:s=0x5599065a7d90:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 run: stack=0x5599065a7d90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 cr:s=0x559906471fe0:op=0x5599065ab800:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 cr:s=0x559906471fe0:op=0x5599065ab800:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 15 stack 0x559906471fe0 end 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 run: stack=0x559906471fe0 is done 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 cr:s=0x5599065a7d90:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 cr:s=0x5599065a7d90:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 trim: trimmed bucket instance dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 15 stack 0x5599065a7d90 end 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 run: stack=0x5599065a7d90 is done 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 4 starting trim on bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065a92e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065a92e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 enqueued request req=0x5599065b3b50 2026-03-20T11:45:57.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 req: 0x5599065b3b50 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fdeb11af900 20 run: stack=0x55990656b1b0 is io blocked 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fde687f8640 20 dequeued request req=0x5599065b3b50 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fde687f8640 20 RGWWQ: empty 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fde687f8640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 : miss 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fde687f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fde687f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fde687f8640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 info.flags=0x17 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fde687f8640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 to cache LRU end 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fde687f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fde687f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fde687f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fde687f8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-39 : miss 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fde687f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fde687f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fde687f8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-39 info.flags=0x0 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fde687f8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-39 to cache LRU end 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fde687f8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-39 : miss 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.628+0000 7fde687f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fde687f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fde687f8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-39 info.flags=0x0 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fde687f8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-39 to cache LRU end 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fde687f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding source pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fde687f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding dest pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fde687f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fde687f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fde687f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065a92e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065a92e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065a92e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065a92e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065b4ff0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 20 run: stack=0x55990656b1b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065794b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065794b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 68f4b319f8f4c5e7f42bfe8036f54bbcf014229fbc242b536d6b3a496ddbfed4 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.631 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.632 INFO:teuthology.orchestra.run.vm02.stderr:68f4b319f8f4c5e7f42bfe8036f54bbcf014229fbc242b536d6b3a496ddbfed4 2026-03-20T11:45:57.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = c68aaec3c5492ed46945810f5dd4afc985bd30b6d920a416e7b124aadc618ce3 2026-03-20T11:45:57.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c68aaec3c5492ed46945810f5dd4afc985bd30b6d920a416e7b124aadc618ce3 2026-03-20T11:45:57.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=11, curl_handle=0x559906573400 2026-03-20T11:45:57.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fdeb11af900 20 run: stack=0x5599065a78d0 is io blocked 2026-03-20T11:45:57.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.629+0000 7fde557fa640 20 link_request req_data=0x559906573b10 req_data->id=11, curl_handle=0x559906573400 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x5599065703a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x5599065703a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x559906570fb0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fdeb11af900 15 stack 0x5599065b41f0 end 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fdeb11af900 20 run: stack=0x5599065b41f0 is done 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065b1290:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fdeb11af900 20 collect(): s=0x559906564fa0 stack=0x5599065b41f0 is complete 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fdeb11af900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fdeb11af900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065b1290:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fdeb11af900 15 stack 0x559906564fa0 end 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fdeb11af900 20 stack->operate() returned ret=-22 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fdeb11af900 20 run: stack=0x559906564fa0 is done 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fdeb11af900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fdeb11af900 4 starting trim on bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065a86c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065a86c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fdeb11af900 20 enqueued request req=0x5599065bcc00 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fdeb11af900 20 req: 0x5599065bcc00 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fdeb11af900 20 run: stack=0x559906564fa0 is io blocked 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fde67ff7640 20 dequeued request req=0x5599065bcc00 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fde67ff7640 20 RGWWQ: empty 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fde67ff7640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 : miss 2026-03-20T11:45:57.661 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fde67ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.662 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fde67ff7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:45:57.662 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fde67ff7640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 info.flags=0x17 2026-03-20T11:45:57.662 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fde67ff7640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 to cache LRU end 2026-03-20T11:45:57.662 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fde67ff7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.662 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fde67ff7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.662 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fde67ff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:45:57.662 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fde67ff7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-8 : miss 2026-03-20T11:45:57.662 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.664+0000 7fde67ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.662 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.662 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fde67ff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.662 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fde67ff7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-8 info.flags=0x0 2026-03-20T11:45:57.662 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fde67ff7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-8 to cache LRU end 2026-03-20T11:45:57.662 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fde67ff7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-8 : miss 2026-03-20T11:45:57.662 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fde67ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fde67ff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fde67ff7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-8 info.flags=0x0 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fde67ff7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-8 to cache LRU end 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fde67ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding source pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fde67ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding dest pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fde67ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fde67ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fde67ff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x5599065aafd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x5599065aafd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065a86c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x5599065734a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 15 stack 0x559906578be0 end 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 run: stack=0x559906578be0 is done 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065a86c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 collect(): s=0x5599065a70d0 stack=0x559906578be0 is complete 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x559906565c00:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 15 stack 0x5599065a70d0 end 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 stack->operate() returned ret=-22 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 run: stack=0x5599065a70d0 is done 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065a86c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065785c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 15 stack 0x5599065b44e0 end 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is done 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065a86c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x559906566ed0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 collect(): s=0x559906565150 stack=0x5599065b44e0 is complete 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 10 trimming bilogs for bucket=:dipepu-55[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15]) markers=[,,,,,00000000006.98.6,,,,,], shards=11 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 4 starting trim on bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065785c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 10 trimming bilog shard 5 of :dipepu-55[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15]) at marker 00000000006.98.6 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 run: stack=0x559906565150 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x5599065a86c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065734a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 run: stack=0x559906564fa0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x5599065a5880:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x5599065a86c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 enqueued request req=0x55990656bce0 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 req: 0x55990656bce0 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 run: stack=0x559906578be0 is io blocked 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065b7160:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fde677f6640 20 dequeued request req=0x55990656bce0 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fde677f6640 20 RGWWQ: empty 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x5599065a5880:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fde677f6640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 : miss 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fde677f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15.5 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 run: stack=0x5599065b41f0 is io blocked 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065b7160:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.665+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fde677f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fde677f6640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 info.flags=0x17 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fde677f6640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 to cache LRU end 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fde677f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fde677f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fde677f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fde677f6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-16 : miss 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fde677f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = e9feba4ac31ff37df629068e603a78dc06bfd92bdc5613fe90a846642efa19b9 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:e9feba4ac31ff37df629068e603a78dc06bfd92bdc5613fe90a846642efa19b9 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fde677f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fde677f6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-16 info.flags=0x0 2026-03-20T11:45:57.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fde677f6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-16 to cache LRU end 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fde677f6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-16 : miss 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fde677f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 413a7750762502e0e6379dbc4d53c4d927bbf4ed973587ed26dec1dd86807ab3 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=413a7750762502e0e6379dbc4d53c4d927bbf4ed973587ed26dec1dd86807ab3 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=12, curl_handle=0x55990656de80 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is io blocked 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fde557fa640 20 link_request req_data=0x55990656e940 req_data->id=12, curl_handle=0x55990656de80 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fde677f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fde677f6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-16 info.flags=0x0 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fde677f6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-16 to cache LRU end 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fde677f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding source pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fde677f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding dest pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fde677f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fde677f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fde677f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x5599065a86c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x5599065a86c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x5599065a86c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x5599065a86c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x55990656a2b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 20 run: stack=0x559906578be0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065aafd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065aafd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 4ea68feaf3d243f3aaa9db88ae3f6b3d3540fab576f539a49207cfe0baeef5c9 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:4ea68feaf3d243f3aaa9db88ae3f6b3d3540fab576f539a49207cfe0baeef5c9 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.666+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.667+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = a9761cb1a9e2753302d54df78fb2dcef1613ad78c54b8340f5a4b39112d97b82 2026-03-20T11:45:57.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.667+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a9761cb1a9e2753302d54df78fb2dcef1613ad78c54b8340f5a4b39112d97b82 2026-03-20T11:45:57.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.667+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.667+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.667+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.667+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=13, curl_handle=0x55990657a950 2026-03-20T11:45:57.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.667+0000 7fdeb11af900 20 run: stack=0x5599065b43a0 is io blocked 2026-03-20T11:45:57.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.667+0000 7fde557fa640 20 link_request req_data=0x55990657b060 req_data->id=13, curl_handle=0x55990657a950 2026-03-20T11:45:57.667 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:57.669+0000 7fd0b73bc640 -1 req 4425761356296570939 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:57.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x5599065a5880:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x5599065a5880:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fdeb11af900 15 stack 0x5599065b41f0 end 2026-03-20T11:45:57.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fdeb11af900 20 run: stack=0x5599065b41f0 is done 2026-03-20T11:45:57.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065785c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x559906566ed0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fdeb11af900 20 trim: trimmed bucket instance dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:45:57.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fdeb11af900 15 stack 0x559906565150 end 2026-03-20T11:45:57.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fdeb11af900 20 run: stack=0x559906565150 is done 2026-03-20T11:45:57.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fdeb11af900 4 starting trim on bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:45:57.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065751b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065751b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fdeb11af900 20 enqueued request req=0x559906572b10 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fdeb11af900 20 req: 0x559906572b10 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fdeb11af900 20 run: stack=0x559906565150 is io blocked 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fde66ff5640 20 dequeued request req=0x559906572b10 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fde66ff5640 20 RGWWQ: empty 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fde66ff5640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 : miss 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fde66ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fde66ff5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=314 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fde66ff5640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 info.flags=0x17 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fde66ff5640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 to cache LRU end 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fde66ff5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fde66ff5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fde66ff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fde66ff5640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-43 : miss 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fde66ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fde66ff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fde66ff5640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-43 info.flags=0x0 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fde66ff5640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-43 to cache LRU end 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fde66ff5640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-43 : miss 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.668+0000 7fde66ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fde66ff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fde66ff5640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-43 info.flags=0x0 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fde66ff5640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-43 to cache LRU end 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fde66ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding source pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fde66ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding dest pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fde66ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fde66ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fde66ff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065751b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065751b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065751b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065751b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x5599065703a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 20 run: stack=0x559906565150 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x5599065c47a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x5599065c47a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 422302dc019e52b68040c7a90d554723833d8f7948591068cb2070bd87f5564e 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:422302dc019e52b68040c7a90d554723833d8f7948591068cb2070bd87f5564e 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 998f533f9d37749f68b5708764726cec15b38a4d07dc4f4166a0c047d79822fb 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=998f533f9d37749f68b5708764726cec15b38a4d07dc4f4166a0c047d79822fb 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=14, curl_handle=0x559906576610 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fdeb11af900 20 run: stack=0x5599065b41f0 is io blocked 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.669+0000 7fde557fa640 20 link_request req_data=0x559906576d60 req_data->id=14, curl_handle=0x559906576610 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.671+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065794b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.671+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065794b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.671+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065794b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.671+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065794b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.671+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065b4ff0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.671+0000 7fdeb11af900 15 stack 0x5599065a78d0 end 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.671+0000 7fdeb11af900 20 run: stack=0x5599065a78d0 is done 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.671+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.671+0000 7fdeb11af900 20 collect(): s=0x55990656b1b0 stack=0x5599065a78d0 is complete 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.671+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.671+0000 7fdeb11af900 10 trimming bilogs for bucket=:dipepu-39[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44]) markers=[,,,,,,,,00000000001.10.6,,], shards=11 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.671+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065785c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.671+0000 7fdeb11af900 10 trimming bilog shard 8 of :dipepu-39[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44]) at marker 00000000001.10.6 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.671+0000 7fdeb11af900 20 run: stack=0x55990656b1b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.671+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065a5880:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.671+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065a5880:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.671+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44.8 2026-03-20T11:45:57.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.671+0000 7fdeb11af900 20 run: stack=0x5599065a78d0 is io blocked 2026-03-20T11:45:57.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065a5880:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065a5880:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fdeb11af900 15 stack 0x5599065a78d0 end 2026-03-20T11:45:57.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fdeb11af900 20 run: stack=0x5599065a78d0 is done 2026-03-20T11:45:57.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065785c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fdeb11af900 20 trim: trimmed bucket instance dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:45:57.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fdeb11af900 15 stack 0x55990656b1b0 end 2026-03-20T11:45:57.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fdeb11af900 20 run: stack=0x55990656b1b0 is done 2026-03-20T11:45:57.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fdeb11af900 4 starting trim on bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:45:57.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065751b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065751b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fdeb11af900 20 enqueued request req=0x5599065b3b50 2026-03-20T11:45:57.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fdeb11af900 20 req: 0x5599065b3b50 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fdeb11af900 20 run: stack=0x55990656b1b0 is io blocked 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fde667f4640 20 dequeued request req=0x5599065b3b50 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fde667f4640 20 RGWWQ: empty 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fde667f4640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 : miss 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fde667f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fde667f4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fde667f4640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 info.flags=0x17 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fde667f4640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 to cache LRU end 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fde667f4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fde667f4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fde667f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fde667f4640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-10 : miss 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fde667f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fde667f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fde667f4640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-10 info.flags=0x0 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fde667f4640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-10 to cache LRU end 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fde667f4640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-10 : miss 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.672+0000 7fde667f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fde667f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fde667f4640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-10 info.flags=0x0 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fde667f4640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-10 to cache LRU end 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fde667f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding source pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fde667f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding dest pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fde667f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fde667f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fde667f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065751b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065751b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065751b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065751b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065b4ff0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 20 run: stack=0x55990656b1b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065bcdf0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065bcdf0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = a3a1976551905c4bc8ae810c328abe183fed71dbcaaa0f7b108f4c2d8b760b38 2026-03-20T11:45:57.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.673 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.673 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.673 INFO:teuthology.orchestra.run.vm02.stderr:a3a1976551905c4bc8ae810c328abe183fed71dbcaaa0f7b108f4c2d8b760b38 2026-03-20T11:45:57.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = bd2f2299289f9bcba953982d1c427a0a5aa30d56bd4d0d7abf5b68152ffa4117 2026-03-20T11:45:57.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=bd2f2299289f9bcba953982d1c427a0a5aa30d56bd4d0d7abf5b68152ffa4117 2026-03-20T11:45:57.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=15, curl_handle=0x559906573400 2026-03-20T11:45:57.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fdeb11af900 20 run: stack=0x5599065a78d0 is io blocked 2026-03-20T11:45:57.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.673+0000 7fde557fa640 20 link_request req_data=0x559906573b10 req_data->id=15, curl_handle=0x559906573400 2026-03-20T11:45:57.705 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065b7160:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.706 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065b7160:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.706 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065aafd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065b7160:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065aafd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065b7160:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065aafd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065734a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 15 stack 0x5599065b44e0 end 2026-03-20T11:45:57.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is done 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065aafd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 collect(): s=0x559906564fa0 stack=0x5599065b44e0 is complete 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 10 trimming bilogs for bucket=:dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) markers=[,00000000002.12.6,,00000000001.5.6,,00000000001.4.6,00000000002.12.6,00000000001.8.6,00000000001.14.6,00000000001.9.6,], shards=11 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x55990656a2b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 15 stack 0x5599065b43a0 end 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 run: stack=0x5599065b43a0 is done 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065c40d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 10 trimming bilog shard 1 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000002.12.6 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 10 trimming bilog shard 3 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.5.6 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 10 trimming bilog shard 5 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.4.6 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 10 trimming bilog shard 6 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000002.12.6 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 10 trimming bilog shard 7 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.8.6 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 10 trimming bilog shard 8 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.14.6 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 10 trimming bilog shard 9 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.9.6 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 run: stack=0x559906564fa0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 collect(): s=0x559906578be0 stack=0x5599065b43a0 is complete 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 10 trimming bilogs for bucket=:dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) markers=[,,,,,00000000003.17.10,,,00000000031.93.9,,], shards=11 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065a5880:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065a90c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x559906571230:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065c2080:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065b3280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 cr:s=0x5599065bdaf0:op=0x5599065ac600:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065bf260:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x5599065734a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 10 trimming bilog shard 5 of :dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) at marker 00000000003.17.10 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 10 trimming bilog shard 8 of :dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) at marker 00000000031.93.9 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 run: stack=0x559906578be0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065a5880:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.709+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.1 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is io blocked 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065a90c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.3 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 run: stack=0x5599065709e0 is io blocked 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x559906571230:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.5 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is io blocked 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065c2080:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.6 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 run: stack=0x559906563410 is io blocked 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065b3280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.7 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 run: stack=0x5599065b6cb0 is io blocked 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 cr:s=0x5599065bdaf0:op=0x5599065ac600:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.8 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 run: stack=0x5599065bdaf0 is io blocked 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065bf260:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.9 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 run: stack=0x5599065a70d0 is io blocked 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065c4f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 cr:s=0x5599065bb0a0:op=0x559906568710:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x5599065c47a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x5599065c47a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:57.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065c4f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21.5 2026-03-20T11:45:57.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 run: stack=0x5599065b43a0 is io blocked 2026-03-20T11:45:57.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 cr:s=0x5599065bb0a0:op=0x559906568710:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21.8 2026-03-20T11:45:57.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 run: stack=0x5599065bb0a0 is io blocked 2026-03-20T11:45:57.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x5599065703a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 15 stack 0x5599065b41f0 end 2026-03-20T11:45:57.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 run: stack=0x5599065b41f0 is done 2026-03-20T11:45:57.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 collect(): s=0x559906565150 stack=0x5599065b41f0 is complete 2026-03-20T11:45:57.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:57.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:57.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:57.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 15 stack 0x559906565150 end 2026-03-20T11:45:57.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 stack->operate() returned ret=-22 2026-03-20T11:45:57.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 run: stack=0x559906565150 is done 2026-03-20T11:45:57.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:57.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 4 starting trim on bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:45:57.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065b7b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065b7b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 enqueued request req=0x559906572b10 2026-03-20T11:45:57.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 req: 0x559906572b10 2026-03-20T11:45:57.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fdeb11af900 20 run: stack=0x559906565150 is io blocked 2026-03-20T11:45:57.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fde65ff3640 20 dequeued request req=0x559906572b10 2026-03-20T11:45:57.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.710+0000 7fde65ff3640 20 RGWWQ: empty 2026-03-20T11:45:57.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.711+0000 7fde65ff3640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 : miss 2026-03-20T11:45:57.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.711+0000 7fde65ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.711+0000 7fde65ff3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:45:57.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.711+0000 7fde65ff3640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 info.flags=0x17 2026-03-20T11:45:57.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.711+0000 7fde65ff3640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 to cache LRU end 2026-03-20T11:45:57.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.711+0000 7fde65ff3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.711+0000 7fde65ff3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.711+0000 7fde65ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:45:57.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.711+0000 7fde65ff3640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-3 : miss 2026-03-20T11:45:57.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.711+0000 7fde65ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.711+0000 7fde65ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.712 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.711+0000 7fde65ff3640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-3 info.flags=0x0 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.711+0000 7fde65ff3640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-3 to cache LRU end 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.711+0000 7fde65ff3640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-3 : miss 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.711+0000 7fde65ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.711+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065a5880:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.711+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065a5880:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.711+0000 7fdeb11af900 15 stack 0x5599065b44e0 end 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.711+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.711+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is done 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.711+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065c40d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.711+0000 7fdeb11af900 20 run: stack=0x559906564fa0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.711+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065b3280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.711+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065b3280:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.711+0000 7fdeb11af900 15 stack 0x5599065b6cb0 end 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.711+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.711+0000 7fdeb11af900 20 run: stack=0x5599065b6cb0 is done 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.711+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065a90c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065a90c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 15 stack 0x5599065709e0 end 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 run: stack=0x5599065709e0 is done 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fde65ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fde65ff3640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-3 info.flags=0x0 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fde65ff3640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-3 to cache LRU end 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fde65ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding source pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fde65ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding dest pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fde65ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fde65ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fde65ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065c40d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065bf260:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065bf260:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 15 stack 0x5599065a70d0 end 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 run: stack=0x5599065a70d0 is done 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065c4f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065c4f10:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 15 stack 0x5599065b43a0 end 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 run: stack=0x5599065b43a0 is done 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065c2080:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065c2080:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 15 stack 0x559906563410 end 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 run: stack=0x559906563410 is done 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x5599065bdaf0:op=0x5599065ac600:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x5599065bdaf0:op=0x5599065ac600:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 15 stack 0x5599065bdaf0 end 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 run: stack=0x5599065bdaf0 is done 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065b7b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x5599065bb0a0:op=0x559906568710:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x5599065bb0a0:op=0x559906568710:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 15 stack 0x5599065bb0a0 end 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 run: stack=0x5599065bb0a0 is done 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065c40d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x559906571230:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x559906571230:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 15 stack 0x5599065ad090 end 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is done 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x5599065734a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065b7b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065c40d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x5599065734a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065b7b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065c40d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x559906578be0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 trim: trimmed bucket instance dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 15 stack 0x559906578be0 end 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 run: stack=0x559906578be0 is done 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065b7b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065c40d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065c40d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 4 starting trim on bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065734a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 run: stack=0x559906565150 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 trim: trimmed bucket instance dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 15 stack 0x559906564fa0 end 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 run: stack=0x559906564fa0 is done 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065bf260:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065c40d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065bf260:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 enqueued request req=0x5599065a76e0 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 req: 0x5599065a76e0 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 run: stack=0x5599065b6cb0 is io blocked 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065c40d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fde657f2640 20 dequeued request req=0x5599065a76e0 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fde657f2640 20 RGWWQ: empty 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fde657f2640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 : miss 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.712+0000 7fde657f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = b5e4a0d86024eea0332841fcfc641cff35e515247231023e1aaf595df4115fc2 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:b5e4a0d86024eea0332841fcfc641cff35e515247231023e1aaf595df4115fc2 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 03bbee9d71a3e5940ab1a0d40e42529a1defc9f6a259bd0bd645a05aeeb8821b 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=03bbee9d71a3e5940ab1a0d40e42529a1defc9f6a259bd0bd645a05aeeb8821b 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=16, curl_handle=0x559906576610 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is io blocked 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fdeb11af900 4 starting trim on bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065a5880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065a5880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fdeb11af900 20 enqueued request req=0x559906564d30 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fdeb11af900 20 req: 0x559906564d30 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde557fa640 20 link_request req_data=0x559906576d60 req_data->id=16, curl_handle=0x559906576610 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde64ff1640 20 dequeued request req=0x559906564d30 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde64ff1640 20 RGWWQ: empty 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fdeb11af900 20 run: stack=0x559906564fa0 is io blocked 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde64ff1640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 : miss 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde64ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde657f2640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde657f2640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 info.flags=0x17 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde657f2640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 to cache LRU end 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde657f2640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde657f2640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde657f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde657f2640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-9 : miss 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde657f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde657f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde657f2640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-9 info.flags=0x0 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde657f2640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-9 to cache LRU end 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde657f2640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-9 : miss 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde657f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde64ff1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde64ff1640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 info.flags=0x17 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde64ff1640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 to cache LRU end 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde64ff1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde64ff1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde64ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde64ff1640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-38 : miss 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde64ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde657f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde657f2640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-9 info.flags=0x0 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde657f2640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-9 to cache LRU end 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde657f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding source pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde657f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding dest pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde657f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde657f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde657f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde64ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde64ff1640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-38 info.flags=0x0 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde64ff1640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-38 to cache LRU end 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde64ff1640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-38 : miss 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fde64ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.713+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065bf260:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fde64ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fde64ff1640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-38 info.flags=0x0 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fde64ff1640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-38 to cache LRU end 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fde64ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding source pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fde64ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding dest pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065bf260:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065bf260:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065bf260:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fde64ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065b99e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fde64ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fde64ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 run: stack=0x5599065b6cb0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065a5880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 408d1aa813ee6348b3f3af18ed687bb7bd061f8a98e00893b8394d35fc057604 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:408d1aa813ee6348b3f3af18ed687bb7bd061f8a98e00893b8394d35fc057604 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 8099b6df29fb07ed8665c23dfde8c0bbec64d439b4cfebf900f3d5a36e76f850 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=8099b6df29fb07ed8665c23dfde8c0bbec64d439b4cfebf900f3d5a36e76f850 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=17, curl_handle=0x55990657a950 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 run: stack=0x5599065a70d0 is io blocked 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065a5880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fde557fa640 20 link_request req_data=0x55990657b060 req_data->id=17, curl_handle=0x55990657a950 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065a5880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065a5880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065c1780:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 run: stack=0x559906564fa0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065a5880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065a5880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.716 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = ce6518d1942bf39a23d901b1e2a53df939be4e3d01623c8f5edbc1b93eab8b18 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr:ce6518d1942bf39a23d901b1e2a53df939be4e3d01623c8f5edbc1b93eab8b18 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 15274b3e14eaeaae1e2709b4012af1f99fbdedfd67c4592ea497217ccf90e9c4 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=15274b3e14eaeaae1e2709b4012af1f99fbdedfd67c4592ea497217ccf90e9c4 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=18, curl_handle=0x55990656de80 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fdeb11af900 20 run: stack=0x5599065b43a0 is io blocked 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.714+0000 7fde557fa640 20 link_request req_data=0x55990656e940 req_data->id=18, curl_handle=0x55990656de80 2026-03-20T11:45:57.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.715+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065bcdf0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.717 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:57.712+0000 7fd093374640 -1 req 203039620114705243 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.715+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065bcdf0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.715+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065bcdf0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.715+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065bcdf0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.715+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065b4ff0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.715+0000 7fdeb11af900 15 stack 0x5599065a78d0 end 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.715+0000 7fdeb11af900 20 run: stack=0x5599065a78d0 is done 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.715+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.716+0000 7fdeb11af900 20 collect(): s=0x55990656b1b0 stack=0x5599065a78d0 is complete 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.716+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.716+0000 7fdeb11af900 10 trimming bilogs for bucket=:dipepu-10[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15]) markers=[,00000000002.9.6,,,,,,,,,], shards=11 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.716+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.716+0000 7fdeb11af900 10 trimming bilog shard 1 of :dipepu-10[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15]) at marker 00000000002.9.6 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.716+0000 7fdeb11af900 20 run: stack=0x55990656b1b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.716+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065a8650:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.716+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065a8650:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.716+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15.1 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.716+0000 7fdeb11af900 20 run: stack=0x5599065a78d0 is io blocked 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065a8650:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065a8650:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fdeb11af900 15 stack 0x5599065a78d0 end 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fdeb11af900 20 run: stack=0x5599065a78d0 is done 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fdeb11af900 20 trim: trimmed bucket instance dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fdeb11af900 15 stack 0x55990656b1b0 end 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fdeb11af900 20 run: stack=0x55990656b1b0 is done 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fdeb11af900 4 starting trim on bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065bf240:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065bf240:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fdeb11af900 20 enqueued request req=0x5599065b3b50 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fdeb11af900 20 req: 0x5599065b3b50 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fdeb11af900 20 run: stack=0x55990656b1b0 is io blocked 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fde63fef640 20 dequeued request req=0x5599065b3b50 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fde63fef640 20 RGWWQ: empty 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fde63fef640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 : miss 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fde63fef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fde63fef640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fde63fef640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 info.flags=0x17 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fde63fef640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 to cache LRU end 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fde63fef640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fde63fef640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fde63fef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fde63fef640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-15 : miss 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.717+0000 7fde63fef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fde63fef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fde63fef640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-15 info.flags=0x0 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fde63fef640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-15 to cache LRU end 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fde63fef640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-15 : miss 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fde63fef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fde63fef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fde63fef640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-15 info.flags=0x0 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fde63fef640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-15 to cache LRU end 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fde63fef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding source pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fde63fef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding dest pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fde63fef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fde63fef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fde63fef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065bf240:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065bf240:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065bf240:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065bf240:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065b4ff0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 20 run: stack=0x55990656b1b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065bf240:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065bf240:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = a5c746c2d6a764d464a54040d216651ece0d10ec1a50e8018e0dbe1278d723ef 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:a5c746c2d6a764d464a54040d216651ece0d10ec1a50e8018e0dbe1278d723ef 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = ebc5a56a874752b527a672ae3297b776b2b28a48f6656ca8b0dad88b09ae929c 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=ebc5a56a874752b527a672ae3297b776b2b28a48f6656ca8b0dad88b09ae929c 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=19, curl_handle=0x559906573400 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fdeb11af900 20 run: stack=0x5599065a78d0 is io blocked 2026-03-20T11:45:57.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.718+0000 7fde557fa640 20 link_request req_data=0x559906573b10 req_data->id=19, curl_handle=0x559906573400 2026-03-20T11:45:57.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065c40d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065c40d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:57.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065734a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fdeb11af900 15 stack 0x5599065ad090 end 2026-03-20T11:45:57.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is done 2026-03-20T11:45:57.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fdeb11af900 20 collect(): s=0x559906565150 stack=0x5599065ad090 is complete 2026-03-20T11:45:57.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fdeb11af900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:57.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fdeb11af900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:57.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:57.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fdeb11af900 15 stack 0x559906565150 end 2026-03-20T11:45:57.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fdeb11af900 20 stack->operate() returned ret=-22 2026-03-20T11:45:57.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fdeb11af900 20 run: stack=0x559906565150 is done 2026-03-20T11:45:57.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fdeb11af900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:57.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fdeb11af900 4 starting trim on bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:45:57.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065b8740:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065b8740:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fdeb11af900 20 enqueued request req=0x559906563140 2026-03-20T11:45:57.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fdeb11af900 20 req: 0x559906563140 2026-03-20T11:45:57.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fdeb11af900 20 run: stack=0x559906565150 is io blocked 2026-03-20T11:45:57.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fde647f0640 20 dequeued request req=0x559906563140 2026-03-20T11:45:57.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fde647f0640 20 RGWWQ: empty 2026-03-20T11:45:57.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fde647f0640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 : miss 2026-03-20T11:45:57.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.754+0000 7fde647f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.755+0000 7fde647f0640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.755+0000 7fde647f0640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 info.flags=0x17 2026-03-20T11:45:57.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.755+0000 7fde647f0640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 to cache LRU end 2026-03-20T11:45:57.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.755+0000 7fde647f0640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.755+0000 7fde647f0640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.755+0000 7fde647f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:45:57.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.755+0000 7fde647f0640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-13 : miss 2026-03-20T11:45:57.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.755+0000 7fde647f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.755+0000 7fde647f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.755+0000 7fde647f0640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-13 info.flags=0x0 2026-03-20T11:45:57.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.755+0000 7fde647f0640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-13 to cache LRU end 2026-03-20T11:45:57.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.755+0000 7fde647f0640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-13 : miss 2026-03-20T11:45:57.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.755+0000 7fde647f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.755+0000 7fde647f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.755+0000 7fde647f0640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-13 info.flags=0x0 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.755+0000 7fde647f0640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-13 to cache LRU end 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.755+0000 7fde647f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18): adding source pipe: {s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.755+0000 7fde647f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18): adding dest pipe: {s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fde647f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fde647f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fde647f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065b8740:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065b8740:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065b8740:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065b8740:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065734a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 20 run: stack=0x559906565150 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065c40d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065c40d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = cb0543f250cb98e8ce8d08be1de06d7054a1de3d0160573b3ba89bb215c7b41d 2026-03-20T11:45:57.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.754 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.754 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.754 INFO:teuthology.orchestra.run.vm02.stderr:cb0543f250cb98e8ce8d08be1de06d7054a1de3d0160573b3ba89bb215c7b41d 2026-03-20T11:45:57.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 895c72dd42aa17ad8cbdac65f38336307f08c7db2c2aa41a935a434dd8a9c077 2026-03-20T11:45:57.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=895c72dd42aa17ad8cbdac65f38336307f08c7db2c2aa41a935a434dd8a9c077 2026-03-20T11:45:57.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=20, curl_handle=0x559906576610 2026-03-20T11:45:57.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is io blocked 2026-03-20T11:45:57.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.756+0000 7fde557fa640 20 link_request req_data=0x559906576d60 req_data->id=20, curl_handle=0x559906576610 2026-03-20T11:45:57.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065a5880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065a5880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065a5880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065b99e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 15 stack 0x5599065a70d0 end 2026-03-20T11:45:57.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 20 run: stack=0x5599065a70d0 is done 2026-03-20T11:45:57.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065a5880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 20 collect(): s=0x5599065b6cb0 stack=0x5599065a70d0 is complete 2026-03-20T11:45:57.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 10 trimming bilogs for bucket=:dipepu-9[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14]) markers=[,00000000002.13.6,,,,,,,,,], shards=11 2026-03-20T11:45:57.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065c1780:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 15 stack 0x5599065b43a0 end 2026-03-20T11:45:57.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 20 run: stack=0x5599065b43a0 is done 2026-03-20T11:45:57.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065cf400:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 10 trimming bilog shard 1 of :dipepu-9[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14]) at marker 00000000002.13.6 2026-03-20T11:45:57.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 20 run: stack=0x5599065b6cb0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 20 collect(): s=0x559906564fa0 stack=0x5599065b43a0 is complete 2026-03-20T11:45:57.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 10 trimming bilogs for bucket=:dipepu-38[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43]) markers=[,,,,,,,,00000000001.38.6,,], shards=11 2026-03-20T11:45:57.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x559906567cf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x55990656a2b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 10 trimming bilog shard 8 of :dipepu-38[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43]) at marker 00000000001.38.6 2026-03-20T11:45:57.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 20 run: stack=0x559906564fa0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x559906567cf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14.1 2026-03-20T11:45:57.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 20 run: stack=0x5599065a70d0 is io blocked 2026-03-20T11:45:57.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065674a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065674a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43.8 2026-03-20T11:45:57.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.758+0000 7fdeb11af900 20 run: stack=0x5599065b43a0 is io blocked 2026-03-20T11:45:57.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x559906567cf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x559906567cf0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 15 stack 0x5599065a70d0 end 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 20 run: stack=0x5599065a70d0 is done 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065cf400:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 20 trim: trimmed bucket instance dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 15 stack 0x5599065b6cb0 end 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 20 run: stack=0x5599065b6cb0 is done 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 4 starting trim on bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x559906567cf0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065674a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065674a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 15 stack 0x5599065b43a0 end 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 20 run: stack=0x5599065b43a0 is done 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x559906567cf0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 20 enqueued request req=0x5599065a76e0 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 20 req: 0x5599065a76e0 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 20 run: stack=0x5599065b6cb0 is io blocked 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x55990656a2b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 20 trim: trimmed bucket instance dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 15 stack 0x559906564fa0 end 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 20 run: stack=0x559906564fa0 is done 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fde637ee640 20 dequeued request req=0x5599065a76e0 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fde637ee640 20 RGWWQ: empty 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fde637ee640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 : miss 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fde637ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.759+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fdeb11af900 4 starting trim on bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065ac600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065ac600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fdeb11af900 20 enqueued request req=0x559906563770 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fdeb11af900 20 req: 0x559906563770 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fdeb11af900 20 run: stack=0x559906564fa0 is io blocked 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde62fed640 20 dequeued request req=0x559906563770 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde62fed640 20 RGWWQ: empty 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde62fed640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 : miss 2026-03-20T11:45:57.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde62fed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde62fed640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde62fed640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 info.flags=0x17 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde62fed640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 to cache LRU end 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde62fed640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde62fed640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde637ee640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde62fed640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde637ee640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 info.flags=0x17 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde637ee640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 to cache LRU end 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde637ee640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde637ee640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde62fed640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-12 : miss 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde637ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde62fed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde637ee640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-11 : miss 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde637ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde62fed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde62fed640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-12 info.flags=0x0 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde62fed640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-12 to cache LRU end 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde62fed640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-12 : miss 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde62fed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde637ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde637ee640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-11 info.flags=0x0 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde637ee640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-11 to cache LRU end 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde637ee640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-11 : miss 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde637ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde637ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde637ee640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-11 info.flags=0x0 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde637ee640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-11 to cache LRU end 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde637ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16): adding source pipe: {s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde637ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16): adding dest pipe: {s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde637ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde637ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde637ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde62fed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde62fed640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-12 info.flags=0x0 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde62fed640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-12 to cache LRU end 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.760+0000 7fde62fed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17): adding source pipe: {s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fde62fed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17): adding dest pipe: {s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fde62fed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fde62fed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fde62fed640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x559906567cf0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x559906567cf0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065ac600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065bf240:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x559906567cf0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065ac600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065bf240:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x559906567cf0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065ac600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065bf240:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065ac600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065bf240:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x55990656a2b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 run: stack=0x5599065b6cb0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065b4ff0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 15 stack 0x5599065a78d0 end 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 run: stack=0x5599065a78d0 is done 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065c9f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065c8cc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 run: stack=0x559906564fa0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 collect(): s=0x55990656b1b0 stack=0x5599065a78d0 is complete 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 10 trimming bilogs for bucket=:dipepu-15[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20]) markers=[,,,,,,,,00000000007.20.6,,], shards=11 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065c9f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = d10e7fe4debaaf015837888496912e11f92ae26c8e73fabfd3726c22bf9d27d7 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:d10e7fe4debaaf015837888496912e11f92ae26c8e73fabfd3726c22bf9d27d7 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 38ee73a2c650d3efa397d9dda08cd1d1d35f9093f789aa0d8562d84217159cfe 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=38ee73a2c650d3efa397d9dda08cd1d1d35f9093f789aa0d8562d84217159cfe 2026-03-20T11:45:57.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=21, curl_handle=0x559906573400 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 run: stack=0x5599065b43a0 is io blocked 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065b4ff0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fde557fa640 20 link_request req_data=0x559906573b10 req_data->id=21, curl_handle=0x559906573400 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065be150:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 10 trimming bilog shard 8 of :dipepu-15[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20]) at marker 00000000007.20.6 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 run: stack=0x55990656b1b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065b4ff0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.761+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.762+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.762+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.762+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 55321adc7e5adcb200d607440e5d62de53163001b44b6b035c915fe5d98d4a4b 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.762+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:55321adc7e5adcb200d607440e5d62de53163001b44b6b035c915fe5d98d4a4b 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.762+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.762+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.762+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.762+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.762+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = bc5bac7b2be8a7fb114599b54f90b16f62d070308eda30f9029b9b9f10f2570c 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.762+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=bc5bac7b2be8a7fb114599b54f90b16f62d070308eda30f9029b9b9f10f2570c 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.762+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.762+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.762+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.762+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=22, curl_handle=0x55990656de80 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.762+0000 7fdeb11af900 20 run: stack=0x5599065a70d0 is io blocked 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.762+0000 7fde557fa640 20 link_request req_data=0x55990656e940 req_data->id=22, curl_handle=0x55990656de80 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.762+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065cd210:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.762+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065cd210:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.762+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20.8 2026-03-20T11:45:57.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.762+0000 7fdeb11af900 20 run: stack=0x5599065a78d0 is io blocked 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.763+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065cd210:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.763+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065cd210:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.763+0000 7fdeb11af900 15 stack 0x5599065a78d0 end 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.763+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.763+0000 7fdeb11af900 20 run: stack=0x5599065a78d0 is done 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.763+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065be150:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.763+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.763+0000 7fdeb11af900 20 trim: trimmed bucket instance dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.763+0000 7fdeb11af900 15 stack 0x55990656b1b0 end 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fdeb11af900 20 run: stack=0x55990656b1b0 is done 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fdeb11af900 4 starting trim on bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065bf240:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065bf240:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fdeb11af900 20 enqueued request req=0x5599065a76e0 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fdeb11af900 20 req: 0x5599065a76e0 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fdeb11af900 20 run: stack=0x55990656b1b0 is io blocked 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fde627ec640 20 dequeued request req=0x5599065a76e0 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fde627ec640 20 RGWWQ: empty 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fde627ec640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 : miss 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fde627ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fde627ec640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fde627ec640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 info.flags=0x17 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fde627ec640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 to cache LRU end 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fde627ec640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fde627ec640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fde627ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fde627ec640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-40 : miss 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fde627ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fde627ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fde627ec640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-40 info.flags=0x0 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fde627ec640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-40 to cache LRU end 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fde627ec640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-40 : miss 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fde627ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fde627ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fde627ec640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-40 info.flags=0x0 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fde627ec640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-40 to cache LRU end 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fde627ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1): adding source pipe: {s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fde627ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1): adding dest pipe: {s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fde627ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fde627ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fde627ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065bf240:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065bf240:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065bf240:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065bf240:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065be150:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.764+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.765+0000 7fdeb11af900 20 run: stack=0x55990656b1b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.765+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065c5970:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.765+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065c5970:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.765+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.765+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.765+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.765+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.764 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.765+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.765+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.765+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 18e7c36dadc095e0d6c148dd47eed90244ca2d1208be82f3ff5b5f7842bc8535 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.765+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:18e7c36dadc095e0d6c148dd47eed90244ca2d1208be82f3ff5b5f7842bc8535 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.765+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.765+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.765+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.765+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.765+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 01c8a2a14c2f3103338c40ea651cce3be9b4d6528900489d3ca9b2c6396cccd8 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.765+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=01c8a2a14c2f3103338c40ea651cce3be9b4d6528900489d3ca9b2c6396cccd8 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.765+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.765+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.765+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.765+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=23, curl_handle=0x55990657a950 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.765+0000 7fdeb11af900 20 run: stack=0x5599065a78d0 is io blocked 2026-03-20T11:45:57.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.765+0000 7fde557fa640 20 link_request req_data=0x55990657b060 req_data->id=23, curl_handle=0x55990657a950 2026-03-20T11:45:57.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.798+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065c40d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.798+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065c40d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.798+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065c40d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.798+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065c40d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.798+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065734a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.798+0000 7fdeb11af900 15 stack 0x5599065ad090 end 2026-03-20T11:45:57.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.798+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is done 2026-03-20T11:45:57.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.798+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.798+0000 7fdeb11af900 20 collect(): s=0x559906565150 stack=0x5599065ad090 is complete 2026-03-20T11:45:57.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.798+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.798+0000 7fdeb11af900 10 trimming bilogs for bucket=:dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) markers=[,,,,,00000000003.29.10,,,00000000003.11.6,00000000011.37.6,], shards=11 2026-03-20T11:45:57.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.798+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x559906571fe0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.798+0000 7fdeb11af900 10 trimming bilog shard 5 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000003.29.10 2026-03-20T11:45:57.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.798+0000 7fdeb11af900 10 trimming bilog shard 8 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000003.11.6 2026-03-20T11:45:57.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.798+0000 7fdeb11af900 10 trimming bilog shard 9 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000011.37.6 2026-03-20T11:45:57.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.798+0000 7fdeb11af900 20 run: stack=0x559906565150 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.798+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065b8740:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.798+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065b3280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.798+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x559906567b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.798+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065b8740:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.798+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.5 2026-03-20T11:45:57.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.798+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is io blocked 2026-03-20T11:45:57.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.798+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065b3280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.798+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.8 2026-03-20T11:45:57.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.798+0000 7fdeb11af900 20 run: stack=0x559906563410 is io blocked 2026-03-20T11:45:57.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.798+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x559906567b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.798+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.9 2026-03-20T11:45:57.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.798+0000 7fdeb11af900 20 run: stack=0x5599065709e0 is io blocked 2026-03-20T11:45:57.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065b8740:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065b8740:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 15 stack 0x5599065ad090 end 2026-03-20T11:45:57.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is done 2026-03-20T11:45:57.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x559906571fe0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 run: stack=0x559906565150 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065b3280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065b3280:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 15 stack 0x559906563410 end 2026-03-20T11:45:57.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 run: stack=0x559906563410 is done 2026-03-20T11:45:57.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x559906571fe0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 run: stack=0x559906565150 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x559906567b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x559906567b70:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 15 stack 0x5599065709e0 end 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 run: stack=0x5599065709e0 is done 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x559906571fe0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 trim: trimmed bucket instance dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 15 stack 0x559906565150 end 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 run: stack=0x559906565150 is done 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 4 starting trim on bucket=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065c8170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065c8170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 enqueued request req=0x559906567470 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 req: 0x559906567470 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fdeb11af900 20 run: stack=0x559906565150 is io blocked 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fde61feb640 20 dequeued request req=0x559906567470 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fde61feb640 20 RGWWQ: empty 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.799+0000 7fde61feb640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 : miss 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fde61feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fde61feb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=387 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fde61feb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 info.flags=0x17 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fde61feb640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 to cache LRU end 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fde61feb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fde61feb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fde61feb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fde61feb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-60 : miss 2026-03-20T11:45:57.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fde61feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fde61feb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fde61feb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-60 info.flags=0x0 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fde61feb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-60 to cache LRU end 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fde61feb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-60 : miss 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fde61feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fde61feb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fde61feb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-60 info.flags=0x0 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fde61feb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-60 to cache LRU end 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fde61feb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20): adding source pipe: {s={b=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fde61feb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20): adding dest pipe: {s={b=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fde61feb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fde61feb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fde61feb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065c8170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065c8170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065c8170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065c8170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065734a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fdeb11af900 20 run: stack=0x559906565150 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x559906571fe0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x559906571fe0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = f256f13e3c0c8c971a99cde26053c16bedc7ac1cb6636bbfa47e26a087b3e543 2026-03-20T11:45:57.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.799 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.799 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.799 INFO:teuthology.orchestra.run.vm02.stderr:f256f13e3c0c8c971a99cde26053c16bedc7ac1cb6636bbfa47e26a087b3e543 2026-03-20T11:45:57.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.800+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.801+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.801+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.801+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = ef4f1d6bd8cd380c132155ad27757d593bf257d08a1425f2cd29737cf532f1a9 2026-03-20T11:45:57.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.801+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=ef4f1d6bd8cd380c132155ad27757d593bf257d08a1425f2cd29737cf532f1a9 2026-03-20T11:45:57.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.801+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.801+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.801+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.801+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=24, curl_handle=0x559906576610 2026-03-20T11:45:57.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.801+0000 7fdeb11af900 20 run: stack=0x5599065709e0 is io blocked 2026-03-20T11:45:57.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.801+0000 7fde557fa640 20 link_request req_data=0x559906576d60 req_data->id=24, curl_handle=0x559906576610 2026-03-20T11:45:57.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065c9f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065c9f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065b4ff0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065c9f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065b4ff0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065c9f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065b4ff0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x55990656a2b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 15 stack 0x5599065b43a0 end 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 run: stack=0x5599065b43a0 is done 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065b4ff0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 collect(): s=0x5599065b6cb0 stack=0x5599065b43a0 is complete 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 10 trimming bilogs for bucket=:dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) markers=[00000000014.52.6,00000000008.34.6,00000000008.50.6,00000000008.32.6,00000000012.28.6,00000000010.31.6,00000000008.21.6,00000000010.27.6,00000000004.27.6,00000000008.22.6,00000000008.19.6], shards=11 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065c8cc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 15 stack 0x5599065a70d0 end 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 run: stack=0x5599065a70d0 is done 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065b99e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 10 trimming bilog shard 0 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000014.52.6 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 10 trimming bilog shard 1 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.34.6 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 10 trimming bilog shard 2 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.50.6 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 10 trimming bilog shard 3 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.32.6 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 10 trimming bilog shard 4 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000012.28.6 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 10 trimming bilog shard 5 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000010.31.6 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 10 trimming bilog shard 6 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.21.6 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 10 trimming bilog shard 7 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000010.27.6 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 10 trimming bilog shard 8 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000004.27.6 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 10 trimming bilog shard 9 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.22.6 2026-03-20T11:45:57.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 10 trimming bilog shard 10 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.19.6 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 run: stack=0x5599065b6cb0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 collect(): s=0x559906564fa0 stack=0x5599065a70d0 is complete 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 10 trimming bilogs for bucket=:dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) markers=[00000000014.44.6,00000000008.52.6,00000000008.40.6,00000000008.21.6,00000000012.26.6,00000000010.26.6,00000000008.35.6,00000000010.32.6,00000000004.38.6,00000000008.22.6,00000000008.23.6], shards=11 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x559906567b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065b3280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065bca80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065a8650:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x5599065d4fb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 cr:s=0x5599065aa250:op=0x5599065aafd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 cr:s=0x559906579650:op=0x5599065d43a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 cr:s=0x5599065d4160:op=0x5599065cb7d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 cr:s=0x5599065c6ef0:op=0x5599065d6110:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 cr:s=0x5599065c1a10:op=0x5599065d2380:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 cr:s=0x5599065d57b0:op=0x5599065ceb90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x55990656a2b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 10 trimming bilog shard 0 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000014.44.6 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 10 trimming bilog shard 1 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.52.6 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 10 trimming bilog shard 2 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.40.6 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 10 trimming bilog shard 3 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.21.6 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 10 trimming bilog shard 4 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000012.26.6 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 10 trimming bilog shard 5 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000010.26.6 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 10 trimming bilog shard 6 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.35.6 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 10 trimming bilog shard 7 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000010.32.6 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 10 trimming bilog shard 8 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000004.38.6 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 10 trimming bilog shard 9 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.22.6 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 10 trimming bilog shard 10 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.23.6 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 run: stack=0x559906564fa0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x559906567b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.0 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 run: stack=0x5599065b43a0 is io blocked 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065b3280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.804+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.1 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.805+0000 7fdeb11af900 20 run: stack=0x559906563410 is io blocked 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.805+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065bca80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.805+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.2 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.805+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is io blocked 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.805+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065a8650:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.805+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.3 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.805+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is io blocked 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.805+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x5599065d4fb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.805+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.4 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.805+0000 7fdeb11af900 20 run: stack=0x5599065b41f0 is io blocked 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.805+0000 7fdeb11af900 20 cr:s=0x5599065aa250:op=0x5599065aafd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.805+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.5 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.805+0000 7fdeb11af900 20 run: stack=0x5599065aa250 is io blocked 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.805+0000 7fdeb11af900 20 cr:s=0x559906579650:op=0x5599065d43a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.805+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.6 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.805+0000 7fdeb11af900 20 run: stack=0x559906579650 is io blocked 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.806+0000 7fdeb11af900 20 cr:s=0x5599065d4160:op=0x5599065cb7d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.806+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.7 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.806+0000 7fdeb11af900 20 run: stack=0x5599065d4160 is io blocked 2026-03-20T11:45:57.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.806+0000 7fdeb11af900 20 cr:s=0x5599065c6ef0:op=0x5599065d6110:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.806+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.8 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.806+0000 7fdeb11af900 20 run: stack=0x5599065c6ef0 is io blocked 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 cr:s=0x5599065c1a10:op=0x5599065d2380:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.9 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 run: stack=0x5599065c1a10 is io blocked 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 cr:s=0x5599065d57b0:op=0x5599065ceb90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.10 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 run: stack=0x5599065d57b0 is io blocked 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065cf390:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065d70d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065d78d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 cr:s=0x5599065c8a20:op=0x5599065d80d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x5599065d88d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 cr:s=0x55990657a4c0:op=0x5599065d90d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 cr:s=0x5599065c0610:op=0x5599065d98d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 cr:s=0x5599065c6c00:op=0x5599065da0d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 cr:s=0x5599065792f0:op=0x5599065da8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 cr:s=0x5599065c9950:op=0x5599065db0d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 cr:s=0x5599065d3f70:op=0x5599065db8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x559906567b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x559906567b70:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 15 stack 0x5599065b43a0 end 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 run: stack=0x5599065b43a0 is done 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065a8650:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065a8650:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 15 stack 0x5599065b44e0 end 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is done 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065bca80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065bca80:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 15 stack 0x5599065ad090 end 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is done 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 cr:s=0x559906579650:op=0x5599065d43a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 cr:s=0x559906579650:op=0x5599065d43a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 15 stack 0x559906579650 end 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 run: stack=0x559906579650 is done 2026-03-20T11:45:57.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065c5970:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 cr:s=0x5599065aa250:op=0x5599065aafd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 cr:s=0x5599065aa250:op=0x5599065aafd0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 15 stack 0x5599065aa250 end 2026-03-20T11:45:57.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 run: stack=0x5599065aa250 is done 2026-03-20T11:45:57.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065b3280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065b3280:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 15 stack 0x559906563410 end 2026-03-20T11:45:57.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.807+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.808+0000 7fdeb11af900 20 run: stack=0x559906563410 is done 2026-03-20T11:45:57.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.808+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x5599065d4fb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.808+0000 7fdeb11af900 20 cr:s=0x5599065b41f0:op=0x5599065d4fb0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.808+0000 7fdeb11af900 15 stack 0x5599065b41f0 end 2026-03-20T11:45:57.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.808+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.808+0000 7fdeb11af900 20 run: stack=0x5599065b41f0 is done 2026-03-20T11:45:57.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.808+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065cf390:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.808+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.0 2026-03-20T11:45:57.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.808+0000 7fdeb11af900 20 run: stack=0x5599065a70d0 is io blocked 2026-03-20T11:45:57.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.808+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065d70d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.808+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.1 2026-03-20T11:45:57.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.809+0000 7fdeb11af900 20 run: stack=0x5599065cfb90 is io blocked 2026-03-20T11:45:57.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.809+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065d78d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.809+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.2 2026-03-20T11:45:57.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.809+0000 7fdeb11af900 20 run: stack=0x559906567970 is io blocked 2026-03-20T11:45:57.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.809+0000 7fdeb11af900 20 cr:s=0x5599065c8a20:op=0x5599065d80d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.809+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.3 2026-03-20T11:45:57.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.809+0000 7fdeb11af900 20 run: stack=0x5599065c8a20 is io blocked 2026-03-20T11:45:57.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.809+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x5599065d88d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.809+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.4 2026-03-20T11:45:57.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.809+0000 7fdeb11af900 20 run: stack=0x5599065d3ce0 is io blocked 2026-03-20T11:45:57.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.809+0000 7fdeb11af900 20 cr:s=0x55990657a4c0:op=0x5599065d90d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.809+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.5 2026-03-20T11:45:57.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.809+0000 7fdeb11af900 20 run: stack=0x55990657a4c0 is io blocked 2026-03-20T11:45:57.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.809+0000 7fdeb11af900 20 cr:s=0x5599065c0610:op=0x5599065d98d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.809+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.6 2026-03-20T11:45:57.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.810+0000 7fdeb11af900 20 run: stack=0x5599065c0610 is io blocked 2026-03-20T11:45:57.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.810+0000 7fdeb11af900 20 cr:s=0x5599065c6c00:op=0x5599065da0d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.810+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.7 2026-03-20T11:45:57.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.810+0000 7fdeb11af900 20 run: stack=0x5599065c6c00 is io blocked 2026-03-20T11:45:57.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.810+0000 7fdeb11af900 20 cr:s=0x5599065792f0:op=0x5599065da8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.810+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.8 2026-03-20T11:45:57.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.810+0000 7fdeb11af900 20 run: stack=0x5599065792f0 is io blocked 2026-03-20T11:45:57.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.810+0000 7fdeb11af900 20 cr:s=0x5599065c9950:op=0x5599065db0d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.810+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.9 2026-03-20T11:45:57.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.810+0000 7fdeb11af900 20 run: stack=0x5599065c9950 is io blocked 2026-03-20T11:45:57.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.810+0000 7fdeb11af900 20 cr:s=0x5599065d3f70:op=0x5599065db8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.810+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.10 2026-03-20T11:45:57.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.810+0000 7fdeb11af900 20 run: stack=0x5599065d3f70 is io blocked 2026-03-20T11:45:57.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.810+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065b99e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.810+0000 7fdeb11af900 20 cr:s=0x5599065d4160:op=0x5599065cb7d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.810+0000 7fdeb11af900 20 cr:s=0x5599065d4160:op=0x5599065cb7d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.810+0000 7fdeb11af900 15 stack 0x5599065d4160 end 2026-03-20T11:45:57.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.810+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.810+0000 7fdeb11af900 20 run: stack=0x5599065d4160 is done 2026-03-20T11:45:57.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065c5970:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 20 cr:s=0x5599065c6ef0:op=0x5599065d6110:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 20 cr:s=0x5599065c6ef0:op=0x5599065d6110:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 15 stack 0x5599065c6ef0 end 2026-03-20T11:45:57.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 20 run: stack=0x5599065c6ef0 is done 2026-03-20T11:45:57.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 20 cr:s=0x5599065d57b0:op=0x5599065ceb90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 20 cr:s=0x5599065d57b0:op=0x5599065ceb90:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 15 stack 0x5599065d57b0 end 2026-03-20T11:45:57.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 20 run: stack=0x5599065d57b0 is done 2026-03-20T11:45:57.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 20 cr:s=0x5599065c1a10:op=0x5599065d2380:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 20 cr:s=0x5599065c1a10:op=0x5599065d2380:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 15 stack 0x5599065c1a10 end 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 20 run: stack=0x5599065c1a10 is done 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065cf390:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065cf390:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 15 stack 0x5599065a70d0 end 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 20 run: stack=0x5599065a70d0 is done 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065d70d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065d70d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 15 stack 0x5599065cfb90 end 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 20 run: stack=0x5599065cfb90 is done 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 20 cr:s=0x5599065c8a20:op=0x5599065d80d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 20 cr:s=0x5599065c8a20:op=0x5599065d80d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 15 stack 0x5599065c8a20 end 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 20 run: stack=0x5599065c8a20 is done 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065d78d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.811+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065d78d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 15 stack 0x559906567970 end 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 run: stack=0x559906567970 is done 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x5599065c0610:op=0x5599065d98d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x5599065c0610:op=0x5599065d98d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 15 stack 0x5599065c0610 end 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 run: stack=0x5599065c0610 is done 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065b99e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065c5970:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x5599065c9950:op=0x5599065db0d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x5599065c9950:op=0x5599065db0d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 15 stack 0x5599065c9950 end 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 run: stack=0x5599065c9950 is done 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x5599065792f0:op=0x5599065da8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x5599065792f0:op=0x5599065da8d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 15 stack 0x5599065792f0 end 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 run: stack=0x5599065792f0 is done 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x55990657a4c0:op=0x5599065d90d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x55990657a4c0:op=0x5599065d90d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 15 stack 0x55990657a4c0 end 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 run: stack=0x55990657a4c0 is done 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x5599065d88d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x5599065d88d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 15 stack 0x5599065d3ce0 end 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 run: stack=0x5599065d3ce0 is done 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x55990656a2b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x5599065d3f70:op=0x5599065db8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x5599065d3f70:op=0x5599065db8d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 15 stack 0x5599065d3f70 end 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 run: stack=0x5599065d3f70 is done 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065b99e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065c5970:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x5599065c6c00:op=0x5599065da0d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x5599065c6c00:op=0x5599065da0d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 15 stack 0x5599065c6c00 end 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 run: stack=0x5599065c6c00 is done 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x55990656a2b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065b99e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x5599065a78d0:op=0x5599065be150:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 15 stack 0x5599065a78d0 end 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 run: stack=0x5599065a78d0 is done 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x55990656a2b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065b99e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 collect(): s=0x55990656b1b0 stack=0x5599065a78d0 is complete 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 10 trimming bilogs for bucket=:dipepu-40[4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1]) markers=[,,,,,00000000001.103.6,,,,,], shards=11 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x55990656a2b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065b99e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065c7060:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 10 trimming bilog shard 5 of :dipepu-40[4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1]) at marker 00000000001.103.6 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 run: stack=0x55990656b1b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x55990656a2b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065b99e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065b3280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x55990656a2b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065b99e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065b3280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1.5 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 run: stack=0x559906567970 is io blocked 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x55990656a2b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065b99e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x55990656a2b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065b99e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x55990656a2b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x5599065b99e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x55990656a2b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x5599065b6cb0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 trim: trimmed bucket instance dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 15 stack 0x5599065b6cb0 end 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 run: stack=0x5599065b6cb0 is done 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x55990656a2b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 trim: trimmed bucket instance dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 15 stack 0x559906564fa0 end 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 run: stack=0x559906564fa0 is done 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.812+0000 7fdeb11af900 4 starting trim on bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x5599065c8170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 4 starting trim on bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x5599065c8170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 enqueued request req=0x559906568710 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 req: 0x559906568710 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 run: stack=0x5599065d3ce0 is io blocked 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065d5950:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde617ea640 20 dequeued request req=0x559906568710 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde617ea640 20 RGWWQ: empty 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde617ea640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 : miss 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde617ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065d5950:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde617ea640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde617ea640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 info.flags=0x17 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde617ea640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 to cache LRU end 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde617ea640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde617ea640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde617ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde617ea640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-18 : miss 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde617ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 enqueued request req=0x5599065b3b50 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 req: 0x5599065b3b50 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 run: stack=0x559906564fa0 is io blocked 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065b3280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065b3280:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde607e8640 20 dequeued request req=0x5599065b3b50 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde607e8640 20 RGWWQ: empty 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 15 stack 0x559906567970 end 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde607e8640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 : miss 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde607e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 run: stack=0x559906567970 is done 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde617ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde617ea640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-18 info.flags=0x0 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde617ea640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-18 to cache LRU end 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde617ea640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-18 : miss 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde617ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x5599065c7060:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 cr:s=0x55990656b1b0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 trim: trimmed bucket instance dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 15 stack 0x55990656b1b0 end 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 run: stack=0x55990656b1b0 is done 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 4 starting trim on bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065b3230:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065b3230:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 enqueued request req=0x559906564d30 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 req: 0x559906564d30 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fdeb11af900 20 run: stack=0x559906567970 is io blocked 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde5ffe7640 20 dequeued request req=0x559906564d30 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde5ffe7640 20 RGWWQ: empty 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde5ffe7640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 : miss 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde5ffe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde617ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde617ea640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-18 info.flags=0x0 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde617ea640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-18 to cache LRU end 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde617ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23): adding source pipe: {s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.813+0000 7fde617ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23): adding dest pipe: {s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde617ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde617ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde617ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x5599065c8170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x5599065c8170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde607e8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde607e8640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 info.flags=0x17 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde607e8640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 to cache LRU end 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde607e8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde607e8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde607e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde607e8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-50 : miss 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde607e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x5599065c8170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.815 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x5599065c8170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065ac600:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 run: stack=0x5599065d3ce0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde5ffe7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde5ffe7640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 info.flags=0x17 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde5ffe7640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 to cache LRU end 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde5ffe7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde5ffe7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde5ffe7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde5ffe7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-24 : miss 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde5ffe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde607e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde607e8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-50 info.flags=0x0 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde607e8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-50 to cache LRU end 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde607e8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-50 : miss 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde607e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 87e6438a6c53b668ecdcfd7031f4f4febc56ea15fd568c4b4c4b0ffe6bfde133 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:87e6438a6c53b668ecdcfd7031f4f4febc56ea15fd568c4b4c4b0ffe6bfde133 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde5ffe7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde5ffe7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-24 info.flags=0x0 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde5ffe7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-24 to cache LRU end 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde5ffe7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-24 : miss 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde5ffe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde607e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde607e8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-50 info.flags=0x0 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde607e8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-50 to cache LRU end 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde607e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10): adding source pipe: {s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde607e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10): adding dest pipe: {s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde607e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde607e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde607e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 3c289daa10329d7abef4f10dc58eb3f70ae7b27238bdbd4f3bd7d3464f566cef 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde5ffe7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde5ffe7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-24 info.flags=0x0 2026-03-20T11:45:57.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde5ffe7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-24 to cache LRU end 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde5ffe7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29): adding source pipe: {s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde5ffe7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29): adding dest pipe: {s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=3c289daa10329d7abef4f10dc58eb3f70ae7b27238bdbd4f3bd7d3464f566cef 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde5ffe7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde5ffe7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde5ffe7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=25, curl_handle=0x55990657a950 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is io blocked 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fde557fa640 20 link_request req_data=0x55990657b060 req_data->id=25, curl_handle=0x55990657a950 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065d5950:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065b3230:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065d5950:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065b3230:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065d5950:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065b3230:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065d5950:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065b3230:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b4ff0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 run: stack=0x559906564fa0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x55990656c620:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 run: stack=0x559906567970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065c5970:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.814+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065cb010:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065c5970:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 0cdb6e869c6ad5ad8f5eba82331efd663d6fcad3ae1803c67788afc86b919601 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:0cdb6e869c6ad5ad8f5eba82331efd663d6fcad3ae1803c67788afc86b919601 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = d25ccf85d7b111368cd84f25e3dd41b400c5e9e39bd131c06b555fb0e94df14c 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=d25ccf85d7b111368cd84f25e3dd41b400c5e9e39bd131c06b555fb0e94df14c 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=26, curl_handle=0x55990656de80 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 20 run: stack=0x5599065cfb90 is io blocked 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065cb010:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fde557fa640 20 link_request req_data=0x55990656e940 req_data->id=26, curl_handle=0x55990656de80 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 96ec5af6b63eded5208f43b6621186b69cb3d52c0a71cf66983fec1b1f362e7e 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:96ec5af6b63eded5208f43b6621186b69cb3d52c0a71cf66983fec1b1f362e7e 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = c15ab0b165c53701a68541825bce07a629ddcf0beb8f633e4ba0bafcff0843c9 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c15ab0b165c53701a68541825bce07a629ddcf0beb8f633e4ba0bafcff0843c9 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=27, curl_handle=0x559906573400 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is io blocked 2026-03-20T11:45:57.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.815+0000 7fde557fa640 20 link_request req_data=0x559906573b10 req_data->id=27, curl_handle=0x559906573400 2026-03-20T11:45:57.818 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:57.815+0000 7fcfdf20c640 -1 req 15854275180825582002 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:57.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x559906571fe0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x559906571fe0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x559906571fe0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x559906571fe0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065734a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 15 stack 0x5599065709e0 end 2026-03-20T11:45:57.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 run: stack=0x5599065709e0 is done 2026-03-20T11:45:57.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 collect(): s=0x559906565150 stack=0x5599065709e0 is complete 2026-03-20T11:45:57.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 10 min_generation is 1 2026-03-20T11:45:57.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065b99e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:57.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 10 bucket_obj is .dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1.0 2026-03-20T11:45:57.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1.0 2026-03-20T11:45:57.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 10 bucket_obj is .dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1.1 2026-03-20T11:45:57.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1.1 2026-03-20T11:45:57.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 10 bucket_obj is .dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1.2 2026-03-20T11:45:57.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1.2 2026-03-20T11:45:57.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 10 bucket_obj is .dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1.3 2026-03-20T11:45:57.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1.3 2026-03-20T11:45:57.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 10 bucket_obj is .dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1.4 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1.4 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 10 bucket_obj is .dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1.5 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1.5 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 10 bucket_obj is .dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1.6 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1.6 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 10 bucket_obj is .dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1.7 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1.7 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 10 bucket_obj is .dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1.8 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1.8 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 10 bucket_obj is .dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1.9 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1.9 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 10 bucket_obj is .dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1.10 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1.10 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 10 bucket_obj is .dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1.11 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1.11 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 10 bucket_obj is .dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1.12 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1.12 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 run: stack=0x559906565150 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065734a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x559906571fe0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065c14a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065d0070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065d4fb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065c6010:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065c6600:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 cr:s=0x5599065ab6c0:op=0x5599065cb7d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 cr:s=0x5599065b5f80:op=0x5599065cbdc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 cr:s=0x5599065ce730:op=0x5599065cc520:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 cr:s=0x5599065ca470:op=0x5599065ccb10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 cr:s=0x5599065b6100:op=0x5599065d6110:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 cr:s=0x5599065cae90:op=0x5599065d6700:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065734a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 run: stack=0x5599065709e0 is io blocked 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x559906571fe0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 run: stack=0x5599065a70d0 is io blocked 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065c14a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 run: stack=0x559906563410 is io blocked 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065d0070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 run: stack=0x5599065b43a0 is io blocked 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.843+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065d4fb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.844+0000 7fdeb11af900 20 run: stack=0x559906567090 is io blocked 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.844+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065c6010:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.842 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.844+0000 7fdeb11af900 20 run: stack=0x5599065639a0 is io blocked 2026-03-20T11:45:57.843 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.844+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065c6600:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.844+0000 7fdeb11af900 20 run: stack=0x5599065cd5d0 is io blocked 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.844+0000 7fdeb11af900 20 cr:s=0x5599065ab6c0:op=0x5599065cb7d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.844+0000 7fdeb11af900 20 run: stack=0x5599065ab6c0 is io blocked 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.844+0000 7fdeb11af900 20 cr:s=0x5599065b5f80:op=0x5599065cbdc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.844+0000 7fdeb11af900 20 run: stack=0x5599065b5f80 is io blocked 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.845+0000 7fdeb11af900 20 cr:s=0x5599065ce730:op=0x5599065cc520:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.845+0000 7fdeb11af900 20 run: stack=0x5599065ce730 is io blocked 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.845+0000 7fdeb11af900 20 cr:s=0x5599065ca470:op=0x5599065ccb10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.845+0000 7fdeb11af900 20 run: stack=0x5599065ca470 is io blocked 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.845+0000 7fdeb11af900 20 cr:s=0x5599065b6100:op=0x5599065d6110:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.845+0000 7fdeb11af900 20 run: stack=0x5599065b6100 is io blocked 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.845+0000 7fdeb11af900 20 cr:s=0x5599065cae90:op=0x5599065d6700:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.845+0000 7fdeb11af900 20 run: stack=0x5599065cae90 is io blocked 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.845+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065c14a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.845+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065c14a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.845+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065c14a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065c14a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 15 stack 0x559906563410 end 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 20 run: stack=0x559906563410 is done 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065b99e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 20 run: stack=0x559906565150 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065734a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065c6600:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065d0070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065c6010:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065734a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065c6600:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x559906571fe0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065d0070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065c6010:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065734a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065c6600:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x559906571fe0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065d0070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065c6010:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065734a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 15 stack 0x5599065709e0 end 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 20 run: stack=0x5599065709e0 is done 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065c6600:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 15 stack 0x5599065cd5d0 end 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 20 run: stack=0x5599065cd5d0 is done 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x559906571fe0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065d0070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 15 stack 0x5599065b43a0 end 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 20 run: stack=0x5599065b43a0 is done 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065c6010:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 15 stack 0x5599065639a0 end 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.846+0000 7fdeb11af900 20 run: stack=0x5599065639a0 is done 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065b99e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065d4fb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x5599065ce730:op=0x5599065cc520:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x5599065ab6c0:op=0x5599065cb7d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x559906571fe0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 15 stack 0x5599065a70d0 end 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 run: stack=0x5599065a70d0 is done 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x5599065b5f80:op=0x5599065cbdc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065b99e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065d4fb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x5599065ce730:op=0x5599065cc520:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x5599065ab6c0:op=0x5599065cb7d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x5599065b5f80:op=0x5599065cbdc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065b99e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065d4fb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x5599065ce730:op=0x5599065cc520:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x5599065ab6c0:op=0x5599065cb7d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x5599065b5f80:op=0x5599065cbdc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065b99e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065d4fb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 15 stack 0x559906567090 end 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 run: stack=0x559906567090 is done 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x5599065ce730:op=0x5599065cc520:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 15 stack 0x5599065ce730 end 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 run: stack=0x5599065ce730 is done 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x5599065ab6c0:op=0x5599065cb7d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 15 stack 0x5599065ab6c0 end 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 run: stack=0x5599065ab6c0 is done 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x5599065b5f80:op=0x5599065cbdc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 15 stack 0x5599065b5f80 end 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 run: stack=0x5599065b5f80 is done 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065b99e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065b99e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065b99e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065b99e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:57.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065b99e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 run: stack=0x559906565150 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x5599065ca470:op=0x5599065ccb10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x5599065ca470:op=0x5599065ccb10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x5599065b6100:op=0x5599065d6110:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x5599065ca470:op=0x5599065ccb10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x5599065b6100:op=0x5599065d6110:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x5599065ca470:op=0x5599065ccb10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 15 stack 0x5599065ca470 end 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 run: stack=0x5599065ca470 is done 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x5599065b6100:op=0x5599065d6110:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065b99e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 run: stack=0x559906565150 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x5599065b6100:op=0x5599065d6110:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 15 stack 0x5599065b6100 end 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 run: stack=0x5599065b6100 is done 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x5599065cae90:op=0x5599065d6700:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065b99e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 run: stack=0x559906565150 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x5599065cae90:op=0x5599065d6700:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x5599065cae90:op=0x5599065d6700:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.847+0000 7fdeb11af900 20 cr:s=0x5599065cae90:op=0x5599065d6700:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.848+0000 7fdeb11af900 15 stack 0x5599065cae90 end 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.848+0000 7fdeb11af900 20 run: stack=0x5599065cae90 is done 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.848+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065b99e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.848+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.848+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065734a0:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.848+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065734a0:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.848+0000 7fdeb11af900 20 enqueued request req=0x5599065c6bf0 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.848+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.848+0000 7fdeb11af900 20 req: 0x5599065c6bf0 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.848+0000 7fdeb11af900 20 run: stack=0x559906565150 is io blocked 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.848+0000 7fde60fe9640 20 dequeued request req=0x5599065c6bf0 2026-03-20T11:45:57.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.848+0000 7fde60fe9640 20 RGWWQ: empty 2026-03-20T11:45:57.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.849+0000 7fde60fe9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 info.flags=0x11 2026-03-20T11:45:57.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.849+0000 7fde60fe9640 10 rgw async rados processor: moving a1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 to cache LRU end 2026-03-20T11:45:57.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.849+0000 7fde60fe9640 10 rgw async rados processor: distributing notification oid=a1.rgw.control:notify.6 cni=[op: 0, obj: a1.rgw.meta:root:.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20, ofs0, ns] 2026-03-20T11:45:57.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.850+0000 7fde8affd640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 343597383929 cookie 94115724918448 notifier 6348 bl.length()=567 2026-03-20T11:45:57.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.850+0000 7fde8affd640 10 rgw watcher librados: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 info.flags=0x11 2026-03-20T11:45:57.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.850+0000 7fde8affd640 10 rgw watcher librados: moving a1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 to cache LRU end 2026-03-20T11:45:57.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.851+0000 7fde60fe9640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-60[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20]): orig_sources= new_sources= 2026-03-20T11:45:57.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.851+0000 7fde60fe9640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-60[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20]): potential sources added=[] removed=[] 2026-03-20T11:45:57.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.851+0000 7fde60fe9640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-60[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20]): orig_dests= new_dests= 2026-03-20T11:45:57.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.851+0000 7fde60fe9640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-60[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20]): potential dests added=[] removed=[] 2026-03-20T11:45:57.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.851+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065734a0:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:57.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.851+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065734a0:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:57.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.851+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065734a0:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:57.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.851+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065734a0:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:45:57.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.851+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.851+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065d5950:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:45:57.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.851+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065d5950:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:45:57.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.851+0000 7fdeb11af900 20 enqueued request req=0x559906097f40 2026-03-20T11:45:57.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.851+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.851+0000 7fdeb11af900 20 req: 0x559906097f40 2026-03-20T11:45:57.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.851+0000 7fdeb11af900 20 run: stack=0x559906565150 is io blocked 2026-03-20T11:45:57.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.851+0000 7fde5efe5640 20 dequeued request req=0x559906097f40 2026-03-20T11:45:57.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.851+0000 7fde5efe5640 20 RGWWQ: empty 2026-03-20T11:45:57.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.852+0000 7fde5efe5640 10 rgw async rados processor: removing a1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 from cache 2026-03-20T11:45:57.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.852+0000 7fde5efe5640 10 rgw async rados processor: distributing notification oid=a1.rgw.control:notify.6 cni=[op: 1, obj: a1.rgw.meta:root:.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20, ofs0, ns] 2026-03-20T11:45:57.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.852+0000 7fde8affd640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 343597383930 cookie 94115724918448 notifier 6348 bl.length()=206 2026-03-20T11:45:57.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065d5950:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:45:57.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065d5950:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:45:57.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065d5950:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:45:57.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065d5950:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:45:57.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fdeb11af900 20 cr:s=0x559906565150:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fdeb11af900 20 trim: trimmed bucket instance dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 2026-03-20T11:45:57.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fdeb11af900 15 stack 0x559906565150 end 2026-03-20T11:45:57.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fdeb11af900 20 run: stack=0x559906565150 is done 2026-03-20T11:45:57.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fdeb11af900 4 starting trim on bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:45:57.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065d5920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065d5920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fdeb11af900 20 enqueued request req=0x55990656e170 2026-03-20T11:45:57.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fdeb11af900 20 req: 0x55990656e170 2026-03-20T11:45:57.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fde5f7e6640 20 dequeued request req=0x55990656e170 2026-03-20T11:45:57.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fde5f7e6640 20 RGWWQ: empty 2026-03-20T11:45:57.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fde5f7e6640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 : miss 2026-03-20T11:45:57.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fde5f7e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fdeb11af900 20 run: stack=0x5599065cd5d0 is io blocked 2026-03-20T11:45:57.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fde5f7e6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fde5f7e6640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 info.flags=0x17 2026-03-20T11:45:57.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fde5f7e6640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 to cache LRU end 2026-03-20T11:45:57.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fde5f7e6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fde5f7e6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fde5f7e6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:45:57.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fde5f7e6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-56 : miss 2026-03-20T11:45:57.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.853+0000 7fde5f7e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fde5f7e6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fde5f7e6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-56 info.flags=0x0 2026-03-20T11:45:57.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fde5f7e6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-56 to cache LRU end 2026-03-20T11:45:57.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fde5f7e6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-56 : miss 2026-03-20T11:45:57.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fde5f7e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fde5f7e6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fde5f7e6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-56 info.flags=0x0 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fde5f7e6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-56 to cache LRU end 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fde5f7e6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16): adding source pipe: {s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fde5f7e6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16): adding dest pipe: {s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fde5f7e6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fde5f7e6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fde5f7e6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065d5920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065d5920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065d5920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065d5920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065c14a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 20 run: stack=0x5599065cd5d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065d5920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065d5920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = ab4bd17f592ecbbe2efd69d33a4bc3dc824171de33c32bd6d5262470c0377900 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:ab4bd17f592ecbbe2efd69d33a4bc3dc824171de33c32bd6d5262470c0377900 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 02da0dc59a6a16b1861547dfc0db8b23bcd43d0def3012cad18b6868294ae1f6 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=02da0dc59a6a16b1861547dfc0db8b23bcd43d0def3012cad18b6868294ae1f6 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=28, curl_handle=0x559906576610 2026-03-20T11:45:57.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fdeb11af900 20 run: stack=0x559906567090 is io blocked 2026-03-20T11:45:57.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.854+0000 7fde557fa640 20 link_request req_data=0x7fde2400cbd0 req_data->id=28, curl_handle=0x559906576610 2026-03-20T11:45:57.853 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:57.854+0000 7fcff0a2f640 -1 req 6773497955748775758 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:57.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065c5970:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065cb010:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065cb010:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:57.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065c5970:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x55990656c620:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 15 stack 0x5599065ad090 end 2026-03-20T11:45:57.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is done 2026-03-20T11:45:57.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065c5970:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 collect(): s=0x559906567970 stack=0x5599065ad090 is complete 2026-03-20T11:45:57.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:57.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:57.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x559906561e70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:57.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 15 stack 0x559906567970 end 2026-03-20T11:45:57.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 stack->operate() returned ret=-22 2026-03-20T11:45:57.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 run: stack=0x559906567970 is done 2026-03-20T11:45:57.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065ac600:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 15 stack 0x5599065b44e0 end 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is done 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065c5970:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 collect(): s=0x5599065d3ce0 stack=0x5599065b44e0 is complete 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 10 trimming bilogs for bucket=:dipepu-18[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23]) markers=[,,,,,,,,00000000005.51.12,,], shards=11 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b4ff0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 15 stack 0x5599065cfb90 end 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 run: stack=0x5599065cfb90 is done 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 4 starting trim on bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x5599065734a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 10 trimming bilog shard 8 of :dipepu-18[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23]) at marker 00000000005.51.12 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 run: stack=0x5599065d3ce0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 collect(): s=0x559906564fa0 stack=0x5599065cfb90 is complete 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065b1270:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 15 stack 0x559906564fa0 end 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 stack->operate() returned ret=-22 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 run: stack=0x559906564fa0 is done 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065b3230:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065be040:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065b3230:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 enqueued request req=0x55990656aaa0 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 req: 0x55990656aaa0 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 run: stack=0x559906567970 is io blocked 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065be040:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fde76ffd640 20 dequeued request req=0x55990656aaa0 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fde76ffd640 20 RGWWQ: empty 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23.8 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fde76ffd640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 : miss 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fde76ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is io blocked 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 4 starting trim on bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065a8600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.857+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065a8600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fdeb11af900 20 enqueued request req=0x559906567470 2026-03-20T11:45:57.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fdeb11af900 20 req: 0x559906567470 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fdeb11af900 20 run: stack=0x5599065cfb90 is io blocked 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde75ffb640 20 dequeued request req=0x559906567470 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde75ffb640 20 RGWWQ: empty 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde75ffb640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 : miss 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde75ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde76ffd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde76ffd640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 info.flags=0x17 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde76ffd640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 to cache LRU end 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde76ffd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde76ffd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde76ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde76ffd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-27 : miss 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde76ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde75ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=314 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde75ffb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 info.flags=0x17 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde75ffb640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 to cache LRU end 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde75ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde75ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde75ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde75ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-44 : miss 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde75ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde76ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde76ffd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-27 info.flags=0x0 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde76ffd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-27 to cache LRU end 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde76ffd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-27 : miss 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde76ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde76ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde76ffd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-27 info.flags=0x0 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde76ffd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-27 to cache LRU end 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde76ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32): adding source pipe: {s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde76ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32): adding dest pipe: {s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde76ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde76ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde76ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde75ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde75ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-44 info.flags=0x0 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde75ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-44 to cache LRU end 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde75ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-44 : miss 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fde75ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065b3230:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065b3230:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065b3230:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065b3230:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065b4ff0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fdeb11af900 20 run: stack=0x559906567970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.858+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fde75ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fde75ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-44 info.flags=0x0 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fde75ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-44 to cache LRU end 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fde75ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4): adding source pipe: {s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fde75ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4): adding dest pipe: {s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fde75ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fde75ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fde75ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.859 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = dc678b01292935e605a08c67c500790e179aadb9702c4caf275a58ea0ed39219 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:dc678b01292935e605a08c67c500790e179aadb9702c4caf275a58ea0ed39219 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = b26527e9d7dd70f2cb3c6108216c9275a9575fa57944a63ae19b5941ddf9f9c2 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b26527e9d7dd70f2cb3c6108216c9275a9575fa57944a63ae19b5941ddf9f9c2 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=29, curl_handle=0x559906573400 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is io blocked 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065a8600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065be040:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065be040:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 15 stack 0x5599065b44e0 end 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is done 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065a8600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x5599065734a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065a8600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 trim: trimmed bucket instance dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 15 stack 0x5599065d3ce0 end 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 run: stack=0x5599065d3ce0 is done 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065a8600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fde557fa640 20 link_request req_data=0x559906573b10 req_data->id=29, curl_handle=0x559906573400 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 4 starting trim on bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065ac600:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 run: stack=0x5599065cfb90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x5599065be040:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x559906571f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x5599065be040:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 enqueued request req=0x5599065b3b50 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 req: 0x5599065b3b50 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 run: stack=0x5599065d3ce0 is io blocked 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x559906571f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fde767fc640 20 dequeued request req=0x5599065b3b50 2026-03-20T11:45:57.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fde767fc640 20 RGWWQ: empty 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fde767fc640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 : miss 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fde767fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 25f7a90cb685de17aef09fffe8ddd70d06ef97becb9d9c9ea7a3b1523640ebae 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fde767fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fde767fc640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 info.flags=0x17 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fde767fc640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 to cache LRU end 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fde767fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fde767fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fde767fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fde767fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-17 : miss 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fde767fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.859+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:25f7a90cb685de17aef09fffe8ddd70d06ef97becb9d9c9ea7a3b1523640ebae 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.860+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.860+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.860+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.860+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.860+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 3946e0eaada085f6a4ea1d60f243c1d759c38a65c2f7226a84d2644427e4fbf2 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.860+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=3946e0eaada085f6a4ea1d60f243c1d759c38a65c2f7226a84d2644427e4fbf2 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.860+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.860+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.860+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.860+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=30, curl_handle=0x55990656de80 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.860+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is io blocked 2026-03-20T11:45:57.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.860+0000 7fde557fa640 20 link_request req_data=0x55990656e940 req_data->id=30, curl_handle=0x55990656de80 2026-03-20T11:45:57.861 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:57.859+0000 7fcfcd1e8640 -1 req 11665636316960340599 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:57.861 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:57.859+0000 7fcfe621a640 -1 req 12505254881418352265 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:57.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.860+0000 7fde767fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.860+0000 7fde767fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-17 info.flags=0x0 2026-03-20T11:45:57.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.860+0000 7fde767fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-17 to cache LRU end 2026-03-20T11:45:57.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.860+0000 7fde767fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-17 : miss 2026-03-20T11:45:57.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.860+0000 7fde767fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fde767fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fde767fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-17 info.flags=0x0 2026-03-20T11:45:57.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fde767fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-17 to cache LRU end 2026-03-20T11:45:57.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fde767fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22): adding source pipe: {s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fde767fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22): adding dest pipe: {s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fde767fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fde767fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fde767fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:45:57.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x5599065be040:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x5599065be040:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x5599065be040:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x5599065be040:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x55990656c620:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 20 run: stack=0x5599065d3ce0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065caff0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065caff0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 13d53d2c7ec5d784e13485945f999841b54c34558106e61b5164a02cea75ca75 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:13d53d2c7ec5d784e13485945f999841b54c34558106e61b5164a02cea75ca75 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = b4d316a1437a867235b12531e5a6eeded0504124c3f1f75d6a097dc33494ce08 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b4d316a1437a867235b12531e5a6eeded0504124c3f1f75d6a097dc33494ce08 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=31, curl_handle=0x55990657a950 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fdeb11af900 20 run: stack=0x5599065639a0 is io blocked 2026-03-20T11:45:57.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.861+0000 7fde557fa640 20 link_request req_data=0x55990657b060 req_data->id=31, curl_handle=0x55990657a950 2026-03-20T11:45:57.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065d5920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065d5920:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:57.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065c14a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fdeb11af900 15 stack 0x559906567090 end 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fdeb11af900 20 run: stack=0x559906567090 is done 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fdeb11af900 20 collect(): s=0x5599065cd5d0 stack=0x559906567090 is complete 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fdeb11af900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fdeb11af900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fdeb11af900 15 stack 0x5599065cd5d0 end 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fdeb11af900 20 stack->operate() returned ret=-22 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fdeb11af900 20 run: stack=0x5599065cd5d0 is done 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fdeb11af900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fdeb11af900 4 starting trim on bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065d58b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065d58b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fdeb11af900 20 enqueued request req=0x55990656e170 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fdeb11af900 20 req: 0x55990656e170 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fdeb11af900 20 run: stack=0x5599065cd5d0 is io blocked 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fde74ff9640 20 dequeued request req=0x55990656e170 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fde74ff9640 20 RGWWQ: empty 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fde74ff9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 : miss 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fde74ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fde74ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fde74ff9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 info.flags=0x17 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fde74ff9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 to cache LRU end 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fde74ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fde74ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fde74ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fde74ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fde74ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-46 : miss 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.896+0000 7fde74ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fde74ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fde74ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-46 info.flags=0x0 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fde74ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-46 to cache LRU end 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fde74ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-46 : miss 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fde74ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fde74ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fde74ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-46 info.flags=0x0 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fde74ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-46 to cache LRU end 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fde74ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6): adding source pipe: {s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fde74ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6): adding dest pipe: {s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fde74ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fde74ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fde74ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065d58b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065d58b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065d58b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065d58b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065734a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 20 run: stack=0x5599065cd5d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065c14a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065c14a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = d51737a8d33d85ba0c6e8055b816774c8d1b35a994e191b1867a2f28951957dc 2026-03-20T11:45:57.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.896 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.896 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.896 INFO:teuthology.orchestra.run.vm02.stderr:d51737a8d33d85ba0c6e8055b816774c8d1b35a994e191b1867a2f28951957dc 2026-03-20T11:45:57.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = a8ac4a37df8cfce55ba7bb119876530d6e8a8bfa665b4d0013eeedf04adaa833 2026-03-20T11:45:57.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a8ac4a37df8cfce55ba7bb119876530d6e8a8bfa665b4d0013eeedf04adaa833 2026-03-20T11:45:57.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=32, curl_handle=0x559906576610 2026-03-20T11:45:57.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fdeb11af900 20 run: stack=0x559906567090 is io blocked 2026-03-20T11:45:57.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.897+0000 7fde557fa640 20 link_request req_data=0x7fde2400cbd0 req_data->id=32, curl_handle=0x559906576610 2026-03-20T11:45:57.896 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:57.897+0000 7fd001250640 -1 req 11860281982726396485 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:57.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:57.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065b4ff0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 15 stack 0x5599065ad090 end 2026-03-20T11:45:57.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is done 2026-03-20T11:45:57.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x559906571f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x559906571f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:57.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 collect(): s=0x559906567970 stack=0x5599065ad090 is complete 2026-03-20T11:45:57.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:57.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:57.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x559906561e70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:57.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 15 stack 0x559906567970 end 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 stack->operate() returned ret=-22 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 run: stack=0x559906567970 is done 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065ac600:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 15 stack 0x5599065b44e0 end 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is done 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 collect(): s=0x5599065cfb90 stack=0x5599065b44e0 is complete 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 15 stack 0x5599065cfb90 end 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 stack->operate() returned ret=-22 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 run: stack=0x5599065cfb90 is done 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 4 starting trim on bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065a8600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 4 starting trim on bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065a8600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 enqueued request req=0x559906563140 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 req: 0x559906563140 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 run: stack=0x559906567970 is io blocked 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b3230:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fde757fa640 20 dequeued request req=0x559906563140 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fde757fa640 20 RGWWQ: empty 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b3230:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 enqueued request req=0x55990656aaa0 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 req: 0x55990656aaa0 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fdeb11af900 20 run: stack=0x5599065cfb90 is io blocked 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fde6b7fe640 20 dequeued request req=0x55990656aaa0 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fde6b7fe640 20 RGWWQ: empty 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fde6b7fe640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 : miss 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fde6b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fde757fa640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 : miss 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fde757fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fde757fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fde757fa640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 info.flags=0x17 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fde757fa640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 to cache LRU end 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fde757fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fde757fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fde757fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fde757fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-37 : miss 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fde757fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fde6b7fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fde6b7fe640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 info.flags=0x17 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fde6b7fe640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 to cache LRU end 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fde6b7fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fde6b7fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fde6b7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fde6b7fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-1 : miss 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.901+0000 7fde6b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fde6b7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fde6b7fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-1 info.flags=0x0 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fde6b7fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-1 to cache LRU end 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fde6b7fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-1 : miss 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fde6b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fde757fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fde757fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-37 info.flags=0x0 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fde757fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-37 to cache LRU end 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fde757fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-37 : miss 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fde757fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fde6b7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fde6b7fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-1 info.flags=0x0 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fde6b7fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-1 to cache LRU end 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fde6b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1): adding source pipe: {s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fde6b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1): adding dest pipe: {s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fde6b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fde6b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fde6b7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b3230:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b3230:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b3230:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b3230:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fde757fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fde757fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-37 info.flags=0x0 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fde757fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-37 to cache LRU end 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fde757fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42): adding source pipe: {s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065ac600:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 run: stack=0x5599065cfb90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fde757fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42): adding dest pipe: {s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fde757fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fde757fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fde757fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = cc14347b6bbd61de7483b230da1f8b8ac5d21d5240b7ab537abbc2022418f9c5 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:cc14347b6bbd61de7483b230da1f8b8ac5d21d5240b7ab537abbc2022418f9c5 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 5eebda45864f77ec1de2dad325c8ae9620dd703d4aab3431e39a9831e233c2fd 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5eebda45864f77ec1de2dad325c8ae9620dd703d4aab3431e39a9831e233c2fd 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=33, curl_handle=0x55990656de80 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is io blocked 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065a8600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065a8600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065a8600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065a8600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fde557fa640 20 link_request req_data=0x55990656e940 req_data->id=33, curl_handle=0x55990656de80 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065b4ff0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 run: stack=0x559906567970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065a5880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065a5880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.902 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.903 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.903 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.903 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.903 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.903 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.903 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.903 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.903 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.903 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 79d10c821059189edf32e5225fbe06a46cced638bb4389f2ade2711033e86f72 2026-03-20T11:45:57.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.903 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.903 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.903 INFO:teuthology.orchestra.run.vm02.stderr:79d10c821059189edf32e5225fbe06a46cced638bb4389f2ade2711033e86f72 2026-03-20T11:45:57.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 92c56737aaa53a2e241fa78bcd84f6aeb34f91e54dc398c573d51898da51bdc9 2026-03-20T11:45:57.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=92c56737aaa53a2e241fa78bcd84f6aeb34f91e54dc398c573d51898da51bdc9 2026-03-20T11:45:57.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=34, curl_handle=0x559906573400 2026-03-20T11:45:57.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is io blocked 2026-03-20T11:45:57.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.902+0000 7fde557fa640 20 link_request req_data=0x559906573b10 req_data->id=34, curl_handle=0x559906573400 2026-03-20T11:45:57.903 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:57.902+0000 7fd01b284640 -1 req 13863647022287269977 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.903+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065caff0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.903+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065caff0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.903+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065caff0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.903+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065caff0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.903+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x55990656c620:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.903+0000 7fdeb11af900 15 stack 0x5599065639a0 end 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.903+0000 7fdeb11af900 20 run: stack=0x5599065639a0 is done 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.903+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.903+0000 7fdeb11af900 20 collect(): s=0x5599065d3ce0 stack=0x5599065639a0 is complete 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.903+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.903+0000 7fdeb11af900 10 trimming bilogs for bucket=:dipepu-17[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22]) markers=[,,,,,,,,00000000005.46.18,,], shards=11 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.903+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x5599065d3310:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.903+0000 7fdeb11af900 10 trimming bilog shard 8 of :dipepu-17[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22]) at marker 00000000005.46.18 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.903+0000 7fdeb11af900 20 run: stack=0x5599065d3ce0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.903+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065cf380:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.903+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065cf380:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.903+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22.8 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fdeb11af900 20 run: stack=0x5599065639a0 is io blocked 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065cf380:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065cf380:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fdeb11af900 15 stack 0x5599065639a0 end 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fdeb11af900 20 run: stack=0x5599065639a0 is done 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x5599065d3310:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fdeb11af900 20 trim: trimmed bucket instance dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fdeb11af900 15 stack 0x5599065d3ce0 end 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fdeb11af900 20 run: stack=0x5599065d3ce0 is done 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fdeb11af900 4 starting trim on bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x5599065cf380:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x5599065cf380:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fdeb11af900 20 enqueued request req=0x5599065b3b50 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fdeb11af900 20 req: 0x5599065b3b50 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fdeb11af900 20 run: stack=0x5599065d3ce0 is io blocked 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fde6bfff640 20 dequeued request req=0x5599065b3b50 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fde6bfff640 20 RGWWQ: empty 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fde6bfff640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 : miss 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fde6bfff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fde6bfff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fde6bfff640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 info.flags=0x17 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fde6bfff640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 to cache LRU end 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fde6bfff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fde6bfff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fde6bfff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fde6bfff640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-7 : miss 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.904+0000 7fde6bfff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fde6bfff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fde6bfff640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-7 info.flags=0x0 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fde6bfff640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-7 to cache LRU end 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fde6bfff640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-7 : miss 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fde6bfff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fde6bfff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fde6bfff640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-7 info.flags=0x0 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fde6bfff640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-7 to cache LRU end 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fde6bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12): adding source pipe: {s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fde6bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12): adding dest pipe: {s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fde6bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fde6bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fde6bfff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x5599065cf380:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x5599065cf380:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x5599065cf380:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x5599065cf380:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065c40a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 20 run: stack=0x5599065d3ce0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x55990656c620:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x55990656c620:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 8bde6642605d10be2df19ec569b4f28a915658d2085ef9fd55a24e8589562c17 2026-03-20T11:45:57.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.906 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.906 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.906 INFO:teuthology.orchestra.run.vm02.stderr:8bde6642605d10be2df19ec569b4f28a915658d2085ef9fd55a24e8589562c17 2026-03-20T11:45:57.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 64173d636fcbf83e1b1e2437e71cfbda94ad8562f61d0c3c98794824a980835c 2026-03-20T11:45:57.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=64173d636fcbf83e1b1e2437e71cfbda94ad8562f61d0c3c98794824a980835c 2026-03-20T11:45:57.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=35, curl_handle=0x55990657a950 2026-03-20T11:45:57.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fdeb11af900 20 run: stack=0x5599065639a0 is io blocked 2026-03-20T11:45:57.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.905+0000 7fde557fa640 20 link_request req_data=0x55990657b060 req_data->id=35, curl_handle=0x55990657a950 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.921+0000 7fde5d7e2640 20 rgw rados thread: process(): notifying mdlog change, shard_id=2 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.921+0000 7fde5d7e2640 20 rgw rados thread: cr:s=0x7fde84003d00:op=0x7fde84003e40:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.921+0000 7fde5d7e2640 20 rgw rados thread: cr:s=0x7fde84003d00:op=0x7fde84003e40:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.921+0000 7fde5d7e2640 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.921+0000 7fde5d7e2640 20 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.921+0000 7fde5d7e2640 20 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.921+0000 7fde5d7e2640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:038966de9f6b9a901b20b4c6ca8b2a46009feebe031babc842d43690c0bc222b 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.921+0000 7fde5d7e2640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]payload request hash = 038966de9f6b9a901b20b4c6ca8b2a46009feebe031babc842d43690c0bc222b 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.921+0000 7fde5d7e2640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]canonical request = POST 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:notify=&rgwx-zonegroup=a&type=metadata 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:038966de9f6b9a901b20b4c6ca8b2a46009feebe031babc842d43690c0bc222b 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:038966de9f6b9a901b20b4c6ca8b2a46009feebe031babc842d43690c0bc222b 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.921+0000 7fde5d7e2640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]canonical request hash = 1aa1d68b576774dee79cb973bd5126b28875b92c4e2b2a3d06b6ab60a8e5795a 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.921+0000 7fde5d7e2640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:1aa1d68b576774dee79cb973bd5126b28875b92c4e2b2a3d06b6ab60a8e5795a 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.921+0000 7fde5d7e2640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.921+0000 7fde5d7e2640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.921+0000 7fde5d7e2640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.921+0000 7fde5d7e2640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.921+0000 7fde5d7e2640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]generated signature = bbd467ab5978c703791b7e76876de6868b2f9a29852e7294e0054815436ee5f1 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.921+0000 7fde5d7e2640 20 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=bbd467ab5978c703791b7e76876de6868b2f9a29852e7294e0054815436ee5f1 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.921+0000 7fde5d7e2640 20 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: 038966de9f6b9a901b20b4c6ca8b2a46009feebe031babc842d43690c0bc222b 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.921+0000 7fde5d7e2640 20 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.921+0000 7fde5d7e2640 20 sending request to http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.921+0000 7fde5d7e2640 20 register_request mgr=0x5599060ef9a8 req_data->id=0, curl_handle=0x7fde84006de0 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.921+0000 7fde5d7e2640 20 run: stack=0x7fde84003d00 is io blocked 2026-03-20T11:45:57.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.921+0000 7fde5dfe3640 20 link_request req_data=0x7fde84007c50 req_data->id=0, curl_handle=0x7fde84006de0 2026-03-20T11:45:57.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.922+0000 7fde5d7e2640 20 rgw rados thread: cr:s=0x7fde84003d00:op=0x7fde84003e40:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:45:57.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.922+0000 7fde5d7e2640 20 rgw rados thread: cr:s=0x7fde84003d00:op=0x7fde84003e40:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:45:57.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.922+0000 7fde5d7e2640 20 rgw rados thread: cr:s=0x7fde84003d00:op=0x7fde84003e40:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:45:57.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.922+0000 7fde5d7e2640 20 rgw rados thread: cr:s=0x7fde84003d00:op=0x7fde84003e40:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:45:57.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.922+0000 7fde5d7e2640 15 stack 0x7fde84003d00 end 2026-03-20T11:45:57.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.922+0000 7fde5d7e2640 20 run: stack=0x7fde84003d00 is done 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065c14a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065c14a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065734a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fdeb11af900 15 stack 0x559906567090 end 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fdeb11af900 20 run: stack=0x559906567090 is done 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fdeb11af900 20 collect(): s=0x5599065cd5d0 stack=0x559906567090 is complete 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fdeb11af900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fdeb11af900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fdeb11af900 15 stack 0x5599065cd5d0 end 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fdeb11af900 20 stack->operate() returned ret=-22 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fdeb11af900 20 run: stack=0x5599065cd5d0 is done 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fdeb11af900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fdeb11af900 4 starting trim on bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065cf380:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065cf380:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fdeb11af900 20 enqueued request req=0x55990656e170 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fdeb11af900 20 req: 0x55990656e170 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fdeb11af900 20 run: stack=0x5599065cd5d0 is io blocked 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fde6affd640 20 dequeued request req=0x55990656e170 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fde6affd640 20 RGWWQ: empty 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fde6affd640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 : miss 2026-03-20T11:45:57.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fde6affd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fde6affd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fde6affd640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 info.flags=0x17 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fde6affd640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 to cache LRU end 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fde6affd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fde6affd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fde6affd640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fde6affd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fde6affd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-48 : miss 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fde6affd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fde6affd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fde6affd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-48 info.flags=0x0 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fde6affd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-48 to cache LRU end 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fde6affd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-48 : miss 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.939+0000 7fde6affd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fde6affd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fde6affd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-48 info.flags=0x0 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fde6affd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-48 to cache LRU end 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fde6affd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8): adding source pipe: {s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fde6affd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8): adding dest pipe: {s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fde6affd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fde6affd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fde6affd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065cf380:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065cf380:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065cf380:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065cf380:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065734a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 20 run: stack=0x5599065cd5d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065c14a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065c14a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.937 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.938 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.938 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.938 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.938 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.938 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.938 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = f47615776ae628339f4f94b9457d8ea40892885b9dcdf8b9ca27117ad99e976e 2026-03-20T11:45:57.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.938 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.938 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.938 INFO:teuthology.orchestra.run.vm02.stderr:f47615776ae628339f4f94b9457d8ea40892885b9dcdf8b9ca27117ad99e976e 2026-03-20T11:45:57.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = db7f9ef98b1cdfbce5e05729f22ae4d1f88d2211777b686bc92233801d7b6de2 2026-03-20T11:45:57.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=db7f9ef98b1cdfbce5e05729f22ae4d1f88d2211777b686bc92233801d7b6de2 2026-03-20T11:45:57.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=36, curl_handle=0x559906576610 2026-03-20T11:45:57.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fdeb11af900 20 run: stack=0x559906567090 is io blocked 2026-03-20T11:45:57.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.940+0000 7fde557fa640 20 link_request req_data=0x7fde2400cbd0 req_data->id=36, curl_handle=0x559906576610 2026-03-20T11:45:57.938 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:57.939+0000 7fd016a7b640 -1 req 12930329175475794341 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:57.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x55990656a2b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:57.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065ac600:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 15 stack 0x5599065b44e0 end 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is done 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065a5880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 collect(): s=0x5599065cfb90 stack=0x5599065b44e0 is complete 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 15 stack 0x5599065cfb90 end 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 stack->operate() returned ret=-22 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 run: stack=0x5599065cfb90 is done 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065a5880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065a5880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 4 starting trim on bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065a5880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065cb7d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065b4ff0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 15 stack 0x5599065ad090 end 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is done 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065cb7d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 enqueued request req=0x5599065c00f0 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 req: 0x5599065c00f0 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fde6a7fc640 20 dequeued request req=0x5599065c00f0 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fde6a7fc640 20 RGWWQ: empty 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fde6a7fc640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 : miss 2026-03-20T11:45:57.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fde6a7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 run: stack=0x5599065cfb90 is io blocked 2026-03-20T11:45:57.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 collect(): s=0x559906567970 stack=0x5599065ad090 is complete 2026-03-20T11:45:57.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 10 trimming bilogs for bucket=:dipepu-37[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42]) markers=[,,,,,,,,,,00000000001.20.6], shards=11 2026-03-20T11:45:57.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x55990656a2b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 10 trimming bilog shard 10 of :dipepu-37[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42]) at marker 00000000001.20.6 2026-03-20T11:45:57.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 run: stack=0x559906567970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065cf380:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065cf380:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42.10 2026-03-20T11:45:57.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is io blocked 2026-03-20T11:45:57.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fde6a7fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fde6a7fc640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 info.flags=0x17 2026-03-20T11:45:57.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fde6a7fc640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 to cache LRU end 2026-03-20T11:45:57.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fde6a7fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fde6a7fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fde6a7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:45:57.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fde6a7fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-59 : miss 2026-03-20T11:45:57.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.944+0000 7fde6a7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fde6a7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fde6a7fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-59 info.flags=0x0 2026-03-20T11:45:57.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fde6a7fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-59 to cache LRU end 2026-03-20T11:45:57.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fde6a7fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-59 : miss 2026-03-20T11:45:57.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fde6a7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065cf380:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065cf380:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 15 stack 0x5599065ad090 end 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is done 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x55990656a2b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 trim: trimmed bucket instance dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 15 stack 0x559906567970 end 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 run: stack=0x559906567970 is done 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 4 starting trim on bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065cf380:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065cf380:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 enqueued request req=0x55990656aaa0 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 req: 0x55990656aaa0 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 run: stack=0x559906567970 is io blocked 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fde69ffb640 20 dequeued request req=0x55990656aaa0 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fde69ffb640 20 RGWWQ: empty 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fde69ffb640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 : miss 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fde69ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fde6a7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fde6a7fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-59 info.flags=0x0 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fde6a7fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-59 to cache LRU end 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fde6a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19): adding source pipe: {s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fde6a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19): adding dest pipe: {s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fde6a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fde6a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fde6a7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065cb7d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065cb7d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065cb7d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065cb7d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065b4ff0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 run: stack=0x5599065cfb90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065a5880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065a5880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 2284e82822077fc78244139e010308870996c82f0ab788e042bfb5a3a6cd1f2e 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:2284e82822077fc78244139e010308870996c82f0ab788e042bfb5a3a6cd1f2e 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 2740ae2bb1c222638aa4f2b102a6588903164a425686895c75f90f12ddc3566d 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2740ae2bb1c222638aa4f2b102a6588903164a425686895c75f90f12ddc3566d 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=37, curl_handle=0x559906573400 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is io blocked 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fde557fa640 20 link_request req_data=0x559906573b10 req_data->id=37, curl_handle=0x559906573400 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fde69ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fde69ffb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 info.flags=0x17 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fde69ffb640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 to cache LRU end 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fde69ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fde69ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fde69ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fde69ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-21 : miss 2026-03-20T11:45:57.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.945+0000 7fde69ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.946+0000 7fde69ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.946+0000 7fde69ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-21 info.flags=0x0 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.946+0000 7fde69ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-21 to cache LRU end 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.946+0000 7fde69ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-21 : miss 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.946+0000 7fde69ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fde69ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fde69ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-21 info.flags=0x0 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fde69ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-21 to cache LRU end 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fde69ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26): adding source pipe: {s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fde69ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26): adding dest pipe: {s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fde69ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fde69ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fde69ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065cf380:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065cf380:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065cf380:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065cf380:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065caff0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 20 run: stack=0x559906567970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065cb7d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065cb7d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.945 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = df0b78aa46b3b6df7e84f801b887e65ab71100338711a5badfb54921a15ade3c 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:df0b78aa46b3b6df7e84f801b887e65ab71100338711a5badfb54921a15ade3c 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 5fe770d1256e98a5e349eaf5eccd2d81512db801608cdbde21e72c5eec85301d 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5fe770d1256e98a5e349eaf5eccd2d81512db801608cdbde21e72c5eec85301d 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=38, curl_handle=0x55990656de80 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is io blocked 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.947+0000 7fde557fa640 20 link_request req_data=0x55990656e940 req_data->id=38, curl_handle=0x55990656de80 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x55990656c620:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x55990656c620:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x55990656c620:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x55990656c620:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065c40a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 15 stack 0x5599065639a0 end 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 run: stack=0x5599065639a0 is done 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 collect(): s=0x5599065d3ce0 stack=0x5599065639a0 is complete 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 10 trimming bilogs for bucket=:dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) markers=[,00000000002.6.6,,00000000001.10.6,,00000000001.6.6,00000000002.6.6,00000000001.6.6,00000000001.12.6,00000000001.4.6,], shards=11 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x559906572030:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 10 trimming bilog shard 1 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000002.6.6 2026-03-20T11:45:57.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 10 trimming bilog shard 3 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.10.6 2026-03-20T11:45:57.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 10 trimming bilog shard 5 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.6.6 2026-03-20T11:45:57.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 10 trimming bilog shard 6 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000002.6.6 2026-03-20T11:45:57.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 10 trimming bilog shard 7 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.6.6 2026-03-20T11:45:57.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 10 trimming bilog shard 8 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.12.6 2026-03-20T11:45:57.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 10 trimming bilog shard 9 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.4.6 2026-03-20T11:45:57.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 run: stack=0x5599065d3ce0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065d2590:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065be040:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065ce730:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599060f99b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065d51f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065c61a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065ca130:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065d2590:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.1 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 run: stack=0x5599065639a0 is io blocked 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065be040:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.3 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 run: stack=0x5599065b43a0 is io blocked 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065ce730:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.5 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 run: stack=0x5599065a70d0 is io blocked 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599060f99b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.6 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 run: stack=0x5599065709e0 is io blocked 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065d51f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.7 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 run: stack=0x559906563410 is io blocked 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065c61a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.8 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 run: stack=0x5599065bb760 is io blocked 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065ca130:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.9 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.948+0000 7fdeb11af900 20 run: stack=0x559906564fa0 is io blocked 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599060f99b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599060f99b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 15 stack 0x5599065709e0 end 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 run: stack=0x5599065709e0 is done 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x559906572030:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 run: stack=0x5599065d3ce0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065d2590:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065d2590:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 15 stack 0x5599065639a0 end 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 run: stack=0x5599065639a0 is done 2026-03-20T11:45:57.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x559906572030:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 run: stack=0x5599065d3ce0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065be040:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065be040:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 15 stack 0x5599065b43a0 end 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 run: stack=0x5599065b43a0 is done 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065d51f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065d51f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 15 stack 0x559906563410 end 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 run: stack=0x559906563410 is done 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065ce730:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065ce730:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 15 stack 0x5599065a70d0 end 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 run: stack=0x5599065a70d0 is done 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065c61a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065c61a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 15 stack 0x5599065bb760 end 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 run: stack=0x5599065bb760 is done 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065ca130:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 cr:s=0x559906564fa0:op=0x5599065ca130:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 15 stack 0x559906564fa0 end 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 run: stack=0x559906564fa0 is done 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x559906572030:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x559906572030:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x559906572030:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x559906572030:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x559906572030:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 cr:s=0x5599065d3ce0:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 trim: trimmed bucket instance dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 15 stack 0x5599065d3ce0 end 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 run: stack=0x5599065d3ce0 is done 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 4 starting trim on bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065d2590:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065d2590:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 enqueued request req=0x5599065a7a50 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 req: 0x5599065a7a50 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fde697fa640 20 dequeued request req=0x5599065a7a50 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fde697fa640 20 RGWWQ: empty 2026-03-20T11:45:57.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fde697fa640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 : miss 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fde697fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.949+0000 7fdeb11af900 20 run: stack=0x5599065bb760 is io blocked 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fde697fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fde697fa640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 info.flags=0x17 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fde697fa640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 to cache LRU end 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fde697fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fde697fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fde697fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fde697fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-41 : miss 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fde697fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fde697fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fde697fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-41 info.flags=0x0 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fde697fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-41 to cache LRU end 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fde697fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-41 : miss 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fde697fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fde697fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fde697fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-41 info.flags=0x0 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fde697fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-41 to cache LRU end 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fde697fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1): adding source pipe: {s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fde697fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1): adding dest pipe: {s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fde697fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fde697fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fde697fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065d2590:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065d2590:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065d2590:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065d2590:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065c40a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 20 run: stack=0x5599065bb760 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065cc4c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065cc4c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = bf0fad469656b45c0835422a6a4117141a6c42254ad34985f171efc3168e4e05 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:bf0fad469656b45c0835422a6a4117141a6c42254ad34985f171efc3168e4e05 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = c036926666e4837aecbf88e0c008c7e88ac7fa42516a0172c029ff1863a1c3ac 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c036926666e4837aecbf88e0c008c7e88ac7fa42516a0172c029ff1863a1c3ac 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=39, curl_handle=0x55990657a950 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fdeb11af900 20 run: stack=0x559906563410 is io blocked 2026-03-20T11:45:57.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.950+0000 7fde557fa640 20 link_request req_data=0x55990657b060 req_data->id=39, curl_handle=0x55990657a950 2026-03-20T11:45:57.951 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:57.950+0000 7fd02eaab640 -1 req 1665195630630711683 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065c14a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065c14a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065734a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 15 stack 0x559906567090 end 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 20 run: stack=0x559906567090 is done 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 20 collect(): s=0x5599065cd5d0 stack=0x559906567090 is complete 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 15 stack 0x5599065cd5d0 end 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 20 stack->operate() returned ret=-22 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 20 run: stack=0x5599065cd5d0 is done 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 4 starting trim on bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065d0070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065d0070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 20 enqueued request req=0x55990656e170 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 20 req: 0x55990656e170 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fde68ff9640 20 dequeued request req=0x55990656e170 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fde68ff9640 20 RGWWQ: empty 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fde68ff9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 : miss 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fde68ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 20 run: stack=0x5599065cd5d0 is io blocked 2026-03-20T11:45:57.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fde68ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:57.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fde68ff9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 info.flags=0x17 2026-03-20T11:45:57.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fde68ff9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 to cache LRU end 2026-03-20T11:45:57.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fde68ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fde68ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fde68ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:45:57.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fde68ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-49 : miss 2026-03-20T11:45:57.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fde68ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fde68ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fde68ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-49 info.flags=0x0 2026-03-20T11:45:57.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fde68ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-49 to cache LRU end 2026-03-20T11:45:57.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fde68ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-49 : miss 2026-03-20T11:45:57.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fde68ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fde68ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fde68ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-49 info.flags=0x0 2026-03-20T11:45:57.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fde68ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-49 to cache LRU end 2026-03-20T11:45:57.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fde68ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9): adding source pipe: {s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fde68ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9): adding dest pipe: {s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fde68ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fde68ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fde68ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:45:57.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065d0070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065d0070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065d0070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065d0070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.981+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.982+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065734a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.982+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.982+0000 7fdeb11af900 20 run: stack=0x5599065cd5d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.982+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065c14a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.982+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065c14a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.982+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.982+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.982+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.982+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.982+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.982+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.982+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = a371cbb002a68fdfa27f0bb6024fcaf1cfaa25fbc80b093a0c6d7b42aed4cd80 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.982+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:a371cbb002a68fdfa27f0bb6024fcaf1cfaa25fbc80b093a0c6d7b42aed4cd80 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.982+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.982+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.982+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.982+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.982+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = fe9f4147a0288673e6731fe9e8ddb3a883c237a3b94d4c4174edd94340ac3eaf 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.982+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=fe9f4147a0288673e6731fe9e8ddb3a883c237a3b94d4c4174edd94340ac3eaf 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.982+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.982+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.982+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.982+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=40, curl_handle=0x559906576610 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.982+0000 7fdeb11af900 20 run: stack=0x559906567090 is io blocked 2026-03-20T11:45:57.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.982+0000 7fde557fa640 20 link_request req_data=0x7fde2400cbd0 req_data->id=40, curl_handle=0x559906576610 2026-03-20T11:45:57.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065a5880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065a5880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065a5880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065a5880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065b4ff0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 15 stack 0x5599065ad090 end 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is done 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 20 collect(): s=0x5599065cfb90 stack=0x5599065ad090 is complete 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 10 trimming bilogs for bucket=:dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) markers=[,00000000001.44.6,,,,00000000001.24.6,,,00000000001.44.6,,00000000001.39.6], shards=11 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 10 trimming bilog shard 1 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.44.6 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 10 trimming bilog shard 5 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.24.6 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 10 trimming bilog shard 8 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.44.6 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 10 trimming bilog shard 10 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.39.6 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 20 run: stack=0x5599065cfb90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065b3230:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065bdfd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065c6170:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065d2590:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065b3230:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.1 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is io blocked 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065bdfd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.5 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 20 run: stack=0x5599065a70d0 is io blocked 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065c6170:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.8 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 20 run: stack=0x5599065b43a0 is io blocked 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065d2590:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.10 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.988+0000 7fdeb11af900 20 run: stack=0x5599065639a0 is io blocked 2026-03-20T11:45:57.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065cb7d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065cb7d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065b3230:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065b3230:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 15 stack 0x5599065ad090 end 2026-03-20T11:45:57.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is done 2026-03-20T11:45:57.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065cb7d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 run: stack=0x5599065cfb90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065cb7d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065caff0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 15 stack 0x5599065b44e0 end 2026-03-20T11:45:57.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is done 2026-03-20T11:45:57.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 collect(): s=0x559906567970 stack=0x5599065b44e0 is complete 2026-03-20T11:45:57.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 10 trimming bilogs for bucket=:dipepu-21[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26]) markers=[,,,,,,,,00000000001.60.14,,], shards=11 2026-03-20T11:45:57.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065caff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 10 trimming bilog shard 8 of :dipepu-21[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26]) at marker 00000000001.60.14 2026-03-20T11:45:57.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 run: stack=0x559906567970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599060fbab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599060fbab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26.8 2026-03-20T11:45:57.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is io blocked 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065bdfd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065bdfd0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 15 stack 0x5599065a70d0 end 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 run: stack=0x5599065a70d0 is done 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 run: stack=0x5599065cfb90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065d2590:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065d2590:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 15 stack 0x5599065639a0 end 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 run: stack=0x5599065639a0 is done 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.989+0000 7fdeb11af900 20 run: stack=0x5599065cfb90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065c6170:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065c6170:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 15 stack 0x5599065b43a0 end 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 run: stack=0x5599065b43a0 is done 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 trim: trimmed bucket instance dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 15 stack 0x5599065cfb90 end 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 run: stack=0x5599065cfb90 is done 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 4 starting trim on bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065bdfd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065bdfd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 enqueued request req=0x55990656bce0 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 req: 0x55990656bce0 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fde687f8640 20 dequeued request req=0x55990656bce0 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fde687f8640 20 RGWWQ: empty 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fde687f8640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 : miss 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fde687f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 run: stack=0x5599065cfb90 is io blocked 2026-03-20T11:45:57.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fde687f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:57.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fde687f8640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 info.flags=0x17 2026-03-20T11:45:57.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fde687f8640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 to cache LRU end 2026-03-20T11:45:57.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fde687f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fde687f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fde687f8640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:45:57.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fde687f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:45:57.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fde687f8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-45 : miss 2026-03-20T11:45:57.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fde687f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599060fbab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:57.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599060fbab0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:57.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 15 stack 0x5599065b44e0 end 2026-03-20T11:45:57.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:57.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is done 2026-03-20T11:45:57.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x5599065caff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:57.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 cr:s=0x559906567970:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 trim: trimmed bucket instance dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:45:57.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 15 stack 0x559906567970 end 2026-03-20T11:45:57.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fde687f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fde687f8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-45 info.flags=0x0 2026-03-20T11:45:57.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fde687f8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-45 to cache LRU end 2026-03-20T11:45:57.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fde687f8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-45 : miss 2026-03-20T11:45:57.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fde687f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 run: stack=0x559906567970 is done 2026-03-20T11:45:57.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 4 starting trim on bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065b3230:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065b3230:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 enqueued request req=0x55990656aaa0 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 req: 0x55990656aaa0 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fde67ff7640 20 dequeued request req=0x55990656aaa0 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fde67ff7640 20 RGWWQ: empty 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fde67ff7640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 : miss 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fde67ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is io blocked 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.990+0000 7fde687f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde687f8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-45 info.flags=0x0 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde687f8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-45 to cache LRU end 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde687f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5): adding source pipe: {s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde687f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5): adding dest pipe: {s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde687f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde687f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde687f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde67ff7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde67ff7640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 info.flags=0x17 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde67ff7640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 to cache LRU end 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde67ff7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde67ff7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde67ff7640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde67ff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde67ff7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-33 : miss 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde67ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065bdfd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065bdfd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065bdfd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065bdfd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065caff0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 run: stack=0x5599065cfb90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065d4cc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065d4cc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.989 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 9af745df735265127a226e48d356e1d1dc274288103128c3a119e92df0ce51f9 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:9af745df735265127a226e48d356e1d1dc274288103128c3a119e92df0ce51f9 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = fe740b4ed50c3556a335dbbf1a08917ef5dda669409df5364644ccdbeb4d9d81 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=fe740b4ed50c3556a335dbbf1a08917ef5dda669409df5364644ccdbeb4d9d81 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=41, curl_handle=0x55990656de80 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 run: stack=0x5599065b43a0 is io blocked 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde557fa640 20 link_request req_data=0x55990656e940 req_data->id=41, curl_handle=0x55990656de80 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde67ff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde67ff7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-33 info.flags=0x0 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde67ff7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-33 to cache LRU end 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde67ff7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-33 : miss 2026-03-20T11:45:57.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde67ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.991 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:57.990+0000 7fd0422d2640 -1 req 15086738882132233388 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:57.991 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:57.991+0000 7fd02028e640 -1 req 9232966562069395558 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:57.991 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:57.993+0000 7fd021290640 -1 req 13082743317110821407 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde67ff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde67ff7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-33 info.flags=0x0 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde67ff7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-33 to cache LRU end 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde67ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38): adding source pipe: {s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde67ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38): adding dest pipe: {s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde67ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde67ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fde67ff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065b3230:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065b3230:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065b3230:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065b3230:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065b4ff0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599060fa850:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599060fa850:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.991+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.992+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.992+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.992+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.992+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.992+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.992+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 69e5b05d8621e87c17f060bc1860c0fffd24d0a8743455d99b5a33430fe131ba 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.992+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:69e5b05d8621e87c17f060bc1860c0fffd24d0a8743455d99b5a33430fe131ba 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.992+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.992+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.992+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.992+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.992+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 758daf35fe6b7cc36addd11d48d3c92a31ea67b9dfc5297d1a3a8515cc8024af 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.992+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=758daf35fe6b7cc36addd11d48d3c92a31ea67b9dfc5297d1a3a8515cc8024af 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.992+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.992+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.992+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.992+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=42, curl_handle=0x559906573400 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.992+0000 7fdeb11af900 20 run: stack=0x5599065639a0 is io blocked 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.992+0000 7fde557fa640 20 link_request req_data=0x559906573b10 req_data->id=42, curl_handle=0x559906573400 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065cc4c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065cc4c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065c40a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 15 stack 0x559906563410 end 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 run: stack=0x559906563410 is done 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 collect(): s=0x5599065bb760 stack=0x559906563410 is complete 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x559906565c00:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 15 stack 0x5599065bb760 end 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 stack->operate() returned ret=-22 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 run: stack=0x5599065bb760 is done 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 4 starting trim on bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 enqueued request req=0x5599065a7a50 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 req: 0x5599065a7a50 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fde677f6640 20 dequeued request req=0x5599065a7a50 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fde677f6640 20 RGWWQ: empty 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fde677f6640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 : miss 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fde677f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 run: stack=0x5599065bb760 is io blocked 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fde677f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fde677f6640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 info.flags=0x17 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fde677f6640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 to cache LRU end 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fde677f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fde677f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fde677f6640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fde677f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fde677f6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-47 : miss 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fde677f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fde677f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fde677f6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-47 info.flags=0x0 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fde677f6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-47 to cache LRU end 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fde677f6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-47 : miss 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fde677f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fde677f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fde677f6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-47 info.flags=0x0 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fde677f6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-47 to cache LRU end 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fde677f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7): adding source pipe: {s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fde677f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7): adding dest pipe: {s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fde677f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fde677f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fde677f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065c40a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 run: stack=0x5599065bb760 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065cb7d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065cb7d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:57.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:57 +0000 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114557Z 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 92733aed9d3770e1781fb639b331d09e7efbbed14764a3ccdf4d9fc5d05567b8 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114557Z 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:92733aed9d3770e1781fb639b331d09e7efbbed14764a3ccdf4d9fc5d05567b8 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = f053d24a4489d119de0441147a6d645bcf97d9d524922018396fdcbc681f9a09 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=f053d24a4489d119de0441147a6d645bcf97d9d524922018396fdcbc681f9a09 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114557Z 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=43, curl_handle=0x55990657a950 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fdeb11af900 20 run: stack=0x559906563410 is io blocked 2026-03-20T11:45:57.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:57.993+0000 7fde557fa640 20 link_request req_data=0x55990657b060 req_data->id=43, curl_handle=0x55990657a950 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065c14a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065c14a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065c14a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065c14a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065734a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fdeb11af900 15 stack 0x559906567090 end 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fdeb11af900 20 run: stack=0x559906567090 is done 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fdeb11af900 20 collect(): s=0x5599065cd5d0 stack=0x559906567090 is complete 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fdeb11af900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fdeb11af900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fdeb11af900 15 stack 0x5599065cd5d0 end 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fdeb11af900 20 stack->operate() returned ret=-22 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fdeb11af900 20 run: stack=0x5599065cd5d0 is done 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fdeb11af900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fdeb11af900 4 starting trim on bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fdeb11af900 20 enqueued request req=0x55990656e170 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fdeb11af900 20 req: 0x55990656e170 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fde66ff5640 20 dequeued request req=0x55990656e170 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fde66ff5640 20 RGWWQ: empty 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fde66ff5640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 : miss 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fde66ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fdeb11af900 20 run: stack=0x5599065cd5d0 is io blocked 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fde66ff5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fde66ff5640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 info.flags=0x17 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fde66ff5640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 to cache LRU end 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fde66ff5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fde66ff5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fde66ff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fde66ff5640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-58 : miss 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fde66ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fde66ff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fde66ff5640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-58 info.flags=0x0 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fde66ff5640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-58 to cache LRU end 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fde66ff5640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-58 : miss 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.023+0000 7fde66ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fde66ff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fde66ff5640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-58 info.flags=0x0 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fde66ff5640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-58 to cache LRU end 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fde66ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18): adding source pipe: {s={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fde66ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18): adding dest pipe: {s={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fde66ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fde66ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fde66ff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065734a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 20 run: stack=0x5599065cd5d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065c14a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065c14a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 22b57d86c0352a24b59db0418ca95cec386116ddfb62f1dcc47d348ff3386830 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114558Z 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:22b57d86c0352a24b59db0418ca95cec386116ddfb62f1dcc47d348ff3386830 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = e2d5978362ed9e3eaa60bfdc4bff60c30c526554b00219f75445a8687bcc7743 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=e2d5978362ed9e3eaa60bfdc4bff60c30c526554b00219f75445a8687bcc7743 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114558Z 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=44, curl_handle=0x559906576610 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fdeb11af900 20 run: stack=0x559906567090 is io blocked 2026-03-20T11:45:58.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.024+0000 7fde557fa640 20 link_request req_data=0x7fde2400cbd0 req_data->id=44, curl_handle=0x559906576610 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065d4cc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065d4cc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065caff0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 15 stack 0x5599065b43a0 end 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 run: stack=0x5599065b43a0 is done 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 collect(): s=0x5599065cfb90 stack=0x5599065b43a0 is complete 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 15 stack 0x5599065cfb90 end 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 stack->operate() returned ret=-22 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 run: stack=0x5599065cfb90 is done 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 4 starting trim on bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 enqueued request req=0x55990656bce0 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 req: 0x55990656bce0 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fde667f4640 20 dequeued request req=0x55990656bce0 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fde667f4640 20 RGWWQ: empty 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fde667f4640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 : miss 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fde667f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 run: stack=0x5599065cfb90 is io blocked 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fde667f4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fde667f4640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 info.flags=0x17 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fde667f4640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 to cache LRU end 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fde667f4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fde667f4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fde667f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fde667f4640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-30 : miss 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fde667f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fde667f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fde667f4640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-30 info.flags=0x0 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fde667f4640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-30 to cache LRU end 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fde667f4640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-30 : miss 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fde667f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fde667f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fde667f4640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-30 info.flags=0x0 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fde667f4640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-30 to cache LRU end 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fde667f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35): adding source pipe: {s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fde667f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35): adding dest pipe: {s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fde667f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fde667f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fde667f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065caff0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 run: stack=0x5599065cfb90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065d4cc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065d4cc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.032+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 2efe3e14aac799768a0fc596e91982e1016d304c64390f4f3fb9da2a32f3f10f 2026-03-20T11:45:58.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114558Z 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2efe3e14aac799768a0fc596e91982e1016d304c64390f4f3fb9da2a32f3f10f 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 20ffaf60d7a1f4c986c0385ff878a86df179a5a95bd4fb45cfb0e518ea481f5d 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=20ffaf60d7a1f4c986c0385ff878a86df179a5a95bd4fb45cfb0e518ea481f5d 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114558Z 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=45, curl_handle=0x55990656de80 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 run: stack=0x5599065b43a0 is io blocked 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fde557fa640 20 link_request req_data=0x55990656e940 req_data->id=45, curl_handle=0x55990656de80 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599060fa850:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599060fa850:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065b4ff0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 15 stack 0x5599065639a0 end 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 run: stack=0x5599065639a0 is done 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 collect(): s=0x5599065b44e0 stack=0x5599065639a0 is complete 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x559906561e70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 15 stack 0x5599065b44e0 end 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 stack->operate() returned ret=-22 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is done 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 4 starting trim on bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 enqueued request req=0x55990656aaa0 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 req: 0x55990656aaa0 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is io blocked 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fde65ff3640 20 dequeued request req=0x55990656aaa0 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fde65ff3640 20 RGWWQ: empty 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fde65ff3640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 : miss 2026-03-20T11:45:58.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fde65ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.031 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:58.032+0000 7fd032ab3640 -1 req 6273139766958624825 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fde65ff3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fde65ff3640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 info.flags=0x17 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fde65ff3640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 to cache LRU end 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fde65ff3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fde65ff3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fde65ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fde65ff3640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-20 : miss 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fde65ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fde65ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fde65ff3640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-20 info.flags=0x0 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fde65ff3640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-20 to cache LRU end 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fde65ff3640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-20 : miss 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fde65ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fde65ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fde65ff3640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-20 info.flags=0x0 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fde65ff3640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-20 to cache LRU end 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fde65ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25): adding source pipe: {s={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fde65ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25): adding dest pipe: {s={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fde65ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fde65ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fde65ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065b4ff0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599060fa850:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599060fa850:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:58.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = dcd346bcc7edc699463dab137049145ce95f66c1de5a2047beb1bcbafa652b26 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114558Z 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:dcd346bcc7edc699463dab137049145ce95f66c1de5a2047beb1bcbafa652b26 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 4f00fc699c554a790c022589e20560a5c9d4f4c7e559d0549a23977a06fa35a3 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=4f00fc699c554a790c022589e20560a5c9d4f4c7e559d0549a23977a06fa35a3 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114558Z 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.033+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.034+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=46, curl_handle=0x559906573400 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.034+0000 7fdeb11af900 20 run: stack=0x5599065639a0 is io blocked 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.034+0000 7fde557fa640 20 link_request req_data=0x559906573b10 req_data->id=46, curl_handle=0x559906573400 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065cb7d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065cb7d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065c40a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 15 stack 0x559906563410 end 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 run: stack=0x559906563410 is done 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 collect(): s=0x5599065bb760 stack=0x559906563410 is complete 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x559906565c00:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 15 stack 0x5599065bb760 end 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 stack->operate() returned ret=-22 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 run: stack=0x5599065bb760 is done 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 4 starting trim on bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 enqueued request req=0x5599065a7a50 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 req: 0x5599065a7a50 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fde657f2640 20 dequeued request req=0x5599065a7a50 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fde657f2640 20 RGWWQ: empty 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fde657f2640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 : miss 2026-03-20T11:45:58.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fde657f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 run: stack=0x5599065bb760 is io blocked 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fde657f2640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fde657f2640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 info.flags=0x17 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fde657f2640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 to cache LRU end 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fde657f2640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fde657f2640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fde657f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fde657f2640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-52 : miss 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fde657f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fde657f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fde657f2640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-52 info.flags=0x0 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fde657f2640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-52 to cache LRU end 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fde657f2640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-52 : miss 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fde657f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fde657f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fde657f2640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-52 info.flags=0x0 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fde657f2640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-52 to cache LRU end 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fde657f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12): adding source pipe: {s={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fde657f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12): adding dest pipe: {s={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fde657f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fde657f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fde657f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065cc370:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065c40a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 run: stack=0x5599065bb760 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065bc940:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065bc940:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.034 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 2a296d6fde3e0e64e58a29c3d7d2c34304b950443ffd7c957c3ef8a7b10bf897 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114558Z 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:2a296d6fde3e0e64e58a29c3d7d2c34304b950443ffd7c957c3ef8a7b10bf897 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.035+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.036+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.036+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 81702be94b862370ce0a40a6e0c6a29feed3f87276ed9debe97c4602f424e0db 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.036+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=81702be94b862370ce0a40a6e0c6a29feed3f87276ed9debe97c4602f424e0db 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.036+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.036+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114558Z 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.036+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.036+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=47, curl_handle=0x55990657a950 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.036+0000 7fdeb11af900 20 run: stack=0x559906563410 is io blocked 2026-03-20T11:45:58.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.036+0000 7fde557fa640 20 link_request req_data=0x55990657b060 req_data->id=47, curl_handle=0x55990657a950 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065c14a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065c14a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065c14a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065c14a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065734a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 15 stack 0x559906567090 end 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 20 run: stack=0x559906567090 is done 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 20 collect(): s=0x5599065cd5d0 stack=0x559906567090 is complete 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 10 trimming bilogs for bucket=:dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) markers=[,00000000001.77.6,,,,00000000001.65.6,,,00000000001.20.6,,00000000001.28.6], shards=11 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065d44c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 10 trimming bilog shard 1 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.77.6 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 10 trimming bilog shard 5 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.65.6 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 10 trimming bilog shard 8 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.20.6 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 10 trimming bilog shard 10 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.28.6 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 20 run: stack=0x5599065cd5d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065d0070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065c8900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065ce730:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065c9eb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065d0070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.1 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 20 run: stack=0x559906567090 is io blocked 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065c8900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.5 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 20 run: stack=0x5599065a70d0 is io blocked 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065ce730:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.8 2026-03-20T11:45:58.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is io blocked 2026-03-20T11:45:58.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065c9eb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.10 2026-03-20T11:45:58.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.066+0000 7fdeb11af900 20 run: stack=0x5599065709e0 is io blocked 2026-03-20T11:45:58.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.067+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065c8900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.067+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599065c8900:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.067+0000 7fdeb11af900 15 stack 0x5599065a70d0 end 2026-03-20T11:45:58.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.067+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.067+0000 7fdeb11af900 20 run: stack=0x5599065a70d0 is done 2026-03-20T11:45:58.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.067+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065d44c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.067+0000 7fdeb11af900 20 run: stack=0x5599065cd5d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.067+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065ce730:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.067+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065ce730:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.067+0000 7fdeb11af900 15 stack 0x5599065ad090 end 2026-03-20T11:45:58.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.067+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.067+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is done 2026-03-20T11:45:58.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.067+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065d44c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.067+0000 7fdeb11af900 20 run: stack=0x5599065cd5d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.067+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065d0070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.067+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065d0070:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.067+0000 7fdeb11af900 15 stack 0x559906567090 end 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.067+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.067+0000 7fdeb11af900 20 run: stack=0x559906567090 is done 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.067+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065d44c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.067+0000 7fdeb11af900 20 run: stack=0x5599065cd5d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065c9eb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065c9eb0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 15 stack 0x5599065709e0 end 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 run: stack=0x5599065709e0 is done 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065d44c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 trim: trimmed bucket instance dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 15 stack 0x5599065cd5d0 end 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 run: stack=0x5599065cd5d0 is done 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 4 starting trim on bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065c6170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065c6170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 enqueued request req=0x5599065c00f0 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 req: 0x5599065c00f0 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fde64ff1640 20 dequeued request req=0x5599065c00f0 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fde64ff1640 20 RGWWQ: empty 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fde64ff1640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 : miss 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fde64ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 run: stack=0x5599065cd5d0 is io blocked 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fde64ff1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:58.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fde64ff1640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 info.flags=0x17 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fde64ff1640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 to cache LRU end 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fde64ff1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fde64ff1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fde64ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fde64ff1640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-42 : miss 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fde64ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fde64ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fde64ff1640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-42 info.flags=0x0 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fde64ff1640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-42 to cache LRU end 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fde64ff1640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-42 : miss 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fde64ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fde64ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fde64ff1640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-42 info.flags=0x0 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fde64ff1640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-42 to cache LRU end 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fde64ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2): adding source pipe: {s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fde64ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2): adding dest pipe: {s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fde64ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fde64ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fde64ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065c6170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065c6170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065c6170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065c6170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065734a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 run: stack=0x5599065cd5d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065c14a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065c14a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 7a2a45ef3e354873e5879a94b91ae0148fcf9439df29d8ff0a317e817f46dfdc 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114558Z 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:7a2a45ef3e354873e5879a94b91ae0148fcf9439df29d8ff0a317e817f46dfdc 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 516f6a87403fc9bafddb098d608c6e30f1163df3b6246d8ab06b045d29049372 2026-03-20T11:45:58.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=516f6a87403fc9bafddb098d608c6e30f1163df3b6246d8ab06b045d29049372 2026-03-20T11:45:58.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114558Z 2026-03-20T11:45:58.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:58.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=48, curl_handle=0x559906576610 2026-03-20T11:45:58.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fdeb11af900 20 run: stack=0x5599065709e0 is io blocked 2026-03-20T11:45:58.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.068+0000 7fde557fa640 20 link_request req_data=0x7fde2400cbd0 req_data->id=48, curl_handle=0x559906576610 2026-03-20T11:45:58.067 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:58.068+0000 7fcfe7a1d640 -1 req 15378694414173565197 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065d4cc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065d4cc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065caff0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 15 stack 0x5599065b43a0 end 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 20 run: stack=0x5599065b43a0 is done 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 20 collect(): s=0x5599065cfb90 stack=0x5599065b43a0 is complete 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 15 stack 0x5599065cfb90 end 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 20 stack->operate() returned ret=-22 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 20 run: stack=0x5599065cfb90 is done 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 4 starting trim on bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065c6170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065c6170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 20 enqueued request req=0x55990656bce0 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 20 RGWWQ: 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 20 req: 0x55990656bce0 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 20 run: stack=0x5599065cfb90 is io blocked 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fde63fef640 20 dequeued request req=0x55990656bce0 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fde63fef640 20 RGWWQ: empty 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fde63fef640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 : miss 2026-03-20T11:45:58.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fde63fef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fde63fef640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fde63fef640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 info.flags=0x17 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fde63fef640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 to cache LRU end 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fde63fef640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fde63fef640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fde63fef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fde63fef640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-2 : miss 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fde63fef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fde63fef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fde63fef640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-2 info.flags=0x0 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fde63fef640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-2 to cache LRU end 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fde63fef640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-2 : miss 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fde63fef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fde63fef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fde63fef640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-2 info.flags=0x0 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fde63fef640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-2 to cache LRU end 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fde63fef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2): adding source pipe: {s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fde63fef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2): adding dest pipe: {s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fde63fef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fde63fef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fde63fef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065c6170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065c6170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065c6170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065c6170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.074+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.075+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065caff0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.075+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.075+0000 7fdeb11af900 20 run: stack=0x5599065cfb90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.075+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065d4cc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.075+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065d4cc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.075+0000 7fdeb11af900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.075+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.075+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.075+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.075+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.075+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.075+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 7ddcb89f59f231035a6cd1717b1cad1418a7eacc6acdd6489963db61467003de 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.075+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114558Z 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:7ddcb89f59f231035a6cd1717b1cad1418a7eacc6acdd6489963db61467003de 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.075+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.075+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:58.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.075+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:58.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.075+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:58.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.075+0000 7fdeb11af900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = e1938c68647751b3db17609fb2677b1bd5fe48d01f399c79fe4d4774ba1f3a0e 2026-03-20T11:45:58.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.075+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=e1938c68647751b3db17609fb2677b1bd5fe48d01f399c79fe4d4774ba1f3a0e 2026-03-20T11:45:58.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.075+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.075+0000 7fdeb11af900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114558Z 2026-03-20T11:45:58.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.075+0000 7fdeb11af900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:58.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.075+0000 7fdeb11af900 20 register_request mgr=0x7fff9af63980 req_data->id=49, curl_handle=0x55990656de80 2026-03-20T11:45:58.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.075+0000 7fdeb11af900 20 run: stack=0x5599065b43a0 is io blocked 2026-03-20T11:45:58.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.075+0000 7fde557fa640 20 link_request req_data=0x55990656e940 req_data->id=49, curl_handle=0x55990656de80 2026-03-20T11:45:58.073 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:58.074+0000 7fd058aff640 -1 req 14870874339918911844 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:58.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.076+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599060fa850:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.076+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599060fa850:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.076+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599060fa850:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.076+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599060fa850:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.076+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065b4ff0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.076+0000 7fdeb11af900 15 stack 0x5599065639a0 end 2026-03-20T11:45:58.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.076+0000 7fdeb11af900 20 run: stack=0x5599065639a0 is done 2026-03-20T11:45:58.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.076+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.076+0000 7fdeb11af900 20 collect(): s=0x5599065b44e0 stack=0x5599065639a0 is complete 2026-03-20T11:45:58.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.076+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:58.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.076+0000 7fdeb11af900 10 trimming bilogs for bucket=:dipepu-20[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25]) markers=[,,,,,,,,00000000005.42.14,,], shards=11 2026-03-20T11:45:58.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.076+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.076+0000 7fdeb11af900 10 trimming bilog shard 8 of :dipepu-20[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25]) at marker 00000000005.42.14 2026-03-20T11:45:58.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.076+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.076+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065b3230:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.076+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065b3230:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.076+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25.8 2026-03-20T11:45:58.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.076+0000 7fdeb11af900 20 run: stack=0x5599065639a0 is io blocked 2026-03-20T11:45:58.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.077+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065b3230:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.077+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065b3230:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.077+0000 7fdeb11af900 15 stack 0x5599065639a0 end 2026-03-20T11:45:58.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.077+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.077+0000 7fdeb11af900 20 run: stack=0x5599065639a0 is done 2026-03-20T11:45:58.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.077+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.077+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x559906561e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.077+0000 7fdeb11af900 20 trim: trimmed bucket instance dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:45:58.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.077+0000 7fdeb11af900 15 stack 0x5599065b44e0 end 2026-03-20T11:45:58.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.077+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is done 2026-03-20T11:45:58.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.077+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:58.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.077+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065bc940:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065bc940:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065bc940:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065bc940:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065c40a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 15 stack 0x559906563410 end 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 run: stack=0x559906563410 is done 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 collect(): s=0x5599065bb760 stack=0x559906563410 is complete 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 10 trimming bilogs for bucket=:dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) markers=[00000000004.19.6,00000000002.34.1,00000000004.47.6,00000000002.22.1,00000000004.31.6,00000000002.67.1,00000000002.32.1,00000000004.16.6,00000000002.26.1,00000000002.76.1,00000000002.54.1], shards=11 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 10 trimming bilog shard 0 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000004.19.6 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 10 trimming bilog shard 1 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.34.1 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 10 trimming bilog shard 2 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000004.47.6 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 10 trimming bilog shard 3 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.22.1 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 10 trimming bilog shard 4 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000004.31.6 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 10 trimming bilog shard 5 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.67.1 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 10 trimming bilog shard 6 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.32.1 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 10 trimming bilog shard 7 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000004.16.6 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 10 trimming bilog shard 8 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.26.1 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 10 trimming bilog shard 9 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.76.1 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 10 trimming bilog shard 10 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.54.1 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 run: stack=0x5599065bb760 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065c7090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065d2590:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065623d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065e88b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065eb900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599060fcdc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 cr:s=0x5599065a8600:op=0x5599060fd5c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 cr:s=0x5599065ceab0:op=0x5599060fb9a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 cr:s=0x5599060fddc0:op=0x5599060fc1a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 cr:s=0x559906572190:op=0x5599060f99b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 cr:s=0x55990656c620:op=0x5599060fa1b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065c7090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.0 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 run: stack=0x559906563410 is io blocked 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065d2590:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.1 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is io blocked 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065623d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.2 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 run: stack=0x5599065639a0 is io blocked 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065e88b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.3 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 run: stack=0x559906567090 is io blocked 2026-03-20T11:45:58.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065eb900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.4 2026-03-20T11:45:58.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is io blocked 2026-03-20T11:45:58.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599060fcdc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.5 2026-03-20T11:45:58.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 run: stack=0x5599065a70d0 is io blocked 2026-03-20T11:45:58.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 cr:s=0x5599065a8600:op=0x5599060fd5c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.6 2026-03-20T11:45:58.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 run: stack=0x5599065a8600 is io blocked 2026-03-20T11:45:58.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 cr:s=0x5599065ceab0:op=0x5599060fb9a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.7 2026-03-20T11:45:58.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 run: stack=0x5599065ceab0 is io blocked 2026-03-20T11:45:58.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 cr:s=0x5599060fddc0:op=0x5599060fc1a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.078+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.8 2026-03-20T11:45:58.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.079+0000 7fdeb11af900 20 run: stack=0x5599060fddc0 is io blocked 2026-03-20T11:45:58.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.079+0000 7fdeb11af900 20 cr:s=0x559906572190:op=0x5599060f99b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.079+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.9 2026-03-20T11:45:58.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.079+0000 7fdeb11af900 20 run: stack=0x559906572190 is io blocked 2026-03-20T11:45:58.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.079+0000 7fdeb11af900 20 cr:s=0x55990656c620:op=0x5599060fa1b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.079+0000 7fdeb11af900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.10 2026-03-20T11:45:58.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.079+0000 7fdeb11af900 20 run: stack=0x55990656c620 is io blocked 2026-03-20T11:45:58.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.079+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065e88b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.079+0000 7fdeb11af900 20 cr:s=0x559906567090:op=0x5599065e88b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.079+0000 7fdeb11af900 15 stack 0x559906567090 end 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 run: stack=0x559906567090 is done 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 run: stack=0x5599065bb760 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065d2590:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x5599065b44e0:op=0x5599065d2590:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 15 stack 0x5599065b44e0 end 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 run: stack=0x5599065b44e0 is done 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x5599065a8600:op=0x5599060fd5c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x5599065a8600:op=0x5599060fd5c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 15 stack 0x5599065a8600 end 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 run: stack=0x5599065a8600 is done 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 run: stack=0x5599065bb760 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x5599060fddc0:op=0x5599060fc1a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x5599060fddc0:op=0x5599060fc1a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 15 stack 0x5599060fddc0 end 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 run: stack=0x5599060fddc0 is done 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 run: stack=0x5599065bb760 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065eb900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x5599065ad090:op=0x5599065eb900:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 15 stack 0x5599065ad090 end 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 run: stack=0x5599065ad090 is done 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 run: stack=0x5599065bb760 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599060fcdc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x5599065a70d0:op=0x5599060fcdc0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 15 stack 0x5599065a70d0 end 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 run: stack=0x5599065a70d0 is done 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 run: stack=0x5599065bb760 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065c7090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x559906563410:op=0x5599065c7090:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 15 stack 0x559906563410 end 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 run: stack=0x559906563410 is done 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 run: stack=0x5599065bb760 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x559906572190:op=0x5599060f99b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x559906572190:op=0x5599060f99b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 15 stack 0x559906572190 end 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 run: stack=0x559906572190 is done 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065623d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x5599065639a0:op=0x5599065623d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 15 stack 0x5599065639a0 end 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 run: stack=0x5599065639a0 is done 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 run: stack=0x5599065bb760 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x55990656c620:op=0x5599060fa1b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x55990656c620:op=0x5599060fa1b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 15 stack 0x55990656c620 end 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 run: stack=0x55990656c620 is done 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 run: stack=0x5599065bb760 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x5599065ceab0:op=0x5599060fb9a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x5599065ceab0:op=0x5599060fb9a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 15 stack 0x5599065ceab0 end 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 run: stack=0x5599065ceab0 is done 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x5599065b4ff0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x5599065bb760:op=0x559906565c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 trim: trimmed bucket instance dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 15 stack 0x5599065bb760 end 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 run: stack=0x5599065bb760 is done 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:58.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.080+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.110+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065c14a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.110+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065c14a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:58.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.110+0000 7fdeb11af900 20 cr:s=0x5599065709e0:op=0x5599065734a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.110+0000 7fdeb11af900 15 stack 0x5599065709e0 end 2026-03-20T11:45:58.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.110+0000 7fdeb11af900 20 run: stack=0x5599065709e0 is done 2026-03-20T11:45:58.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.110+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.110+0000 7fdeb11af900 20 collect(): s=0x5599065cd5d0 stack=0x5599065709e0 is complete 2026-03-20T11:45:58.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.110+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:58.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.110+0000 7fdeb11af900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:58.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.110+0000 7fdeb11af900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:58.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.110+0000 7fdeb11af900 20 cr:s=0x5599065cd5d0:op=0x5599065ae7f0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:58.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.110+0000 7fdeb11af900 15 stack 0x5599065cd5d0 end 2026-03-20T11:45:58.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.110+0000 7fdeb11af900 20 stack->operate() returned ret=-22 2026-03-20T11:45:58.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.110+0000 7fdeb11af900 20 run: stack=0x5599065cd5d0 is done 2026-03-20T11:45:58.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.110+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:58.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.110+0000 7fdeb11af900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:58.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.110+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.116+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065d4cc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.116+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065d4cc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:58.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.116+0000 7fdeb11af900 20 cr:s=0x5599065b43a0:op=0x5599065caff0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.116+0000 7fdeb11af900 15 stack 0x5599065b43a0 end 2026-03-20T11:45:58.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.116+0000 7fdeb11af900 20 run: stack=0x5599065b43a0 is done 2026-03-20T11:45:58.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.116+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.116+0000 7fdeb11af900 20 collect(): s=0x5599065cfb90 stack=0x5599065b43a0 is complete 2026-03-20T11:45:58.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.116+0000 7fdeb11af900 10 min_generation is 0 2026-03-20T11:45:58.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.116+0000 7fdeb11af900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:58.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.116+0000 7fdeb11af900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:58.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.116+0000 7fdeb11af900 20 cr:s=0x5599065cfb90:op=0x5599065b1270:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:58.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.116+0000 7fdeb11af900 15 stack 0x5599065cfb90 end 2026-03-20T11:45:58.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.116+0000 7fdeb11af900 20 stack->operate() returned ret=-22 2026-03-20T11:45:58.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.116+0000 7fdeb11af900 20 run: stack=0x5599065cfb90 is done 2026-03-20T11:45:58.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.116+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:58.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.116+0000 7fdeb11af900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:58.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.116+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x559906564750:27BucketTrimInstanceCollectCR: operate() returned r=-22 2026-03-20T11:45:58.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.116+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x5599065603d0:12BucketTrimCR: operate() 2026-03-20T11:45:58.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.116+0000 7fdeb11af900 20 writing bucket trim marker=12:b497a80d:root::.bucket.meta.dipepu-57%3a4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17:head 2026-03-20T11:45:58.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.116+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x5599065ead70:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:58.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.116+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x5599065ead70:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:58.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.116+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is io blocked 2026-03-20T11:45:58.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.117+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x5599065ead70:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:58.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.117+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x5599065ead70:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:58.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.117+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x5599065ead70:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:58.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.117+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x5599065ead70:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:58.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.117+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x5599065603d0:12BucketTrimCR: operate() 2026-03-20T11:45:58.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.117+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x5599065d6780:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:58.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.117+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x5599065d6780:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:58.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.117+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is io blocked 2026-03-20T11:45:58.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.118+0000 7fde8affd640 20 trim: bucket trim completed 2026-03-20T11:45:58.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.118+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x5599065d6780:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:58.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.118+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x5599065d6780:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:58.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.118+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x5599065d6780:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:58.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.118+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x5599065d6780:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:58.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.118+0000 7fdeb11af900 20 cr:s=0x559906560ad0:op=0x5599065603d0:12BucketTrimCR: operate() 2026-03-20T11:45:58.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.118+0000 7fdeb11af900 4 bucket index log processing completed in 0.749364197s 2026-03-20T11:45:58.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.118+0000 7fdeb11af900 15 stack 0x559906560ad0 end 2026-03-20T11:45:58.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.118+0000 7fdeb11af900 20 run: stack=0x559906560ad0 is done 2026-03-20T11:45:58.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.123+0000 7fdeb11af900 20 remove_watcher() i=4 2026-03-20T11:45:58.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.123+0000 7fdeb11af900 2 removed watcher, disabling cache 2026-03-20T11:45:58.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.124+0000 7fdeb11af900 20 remove_watcher() i=1 2026-03-20T11:45:58.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.124+0000 7fdeb11af900 20 remove_watcher() i=2 2026-03-20T11:45:58.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.124+0000 7fdeb11af900 20 remove_watcher() i=7 2026-03-20T11:45:58.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.124+0000 7fdeb11af900 20 remove_watcher() i=3 2026-03-20T11:45:58.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.124+0000 7fdeb11af900 20 remove_watcher() i=5 2026-03-20T11:45:58.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.124+0000 7fdeb11af900 20 remove_watcher() i=0 2026-03-20T11:45:58.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.124+0000 7fdeb11af900 20 remove_watcher() i=6 2026-03-20T11:45:58.250 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:58.250 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:58.250 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:45:58.288 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:58.288 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:58.361 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:45:58.370 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774007137.857782_141.1","last_update":"2026-03-20T11:45:37.857782Z"},{"marker":"1_1774007137.912792_174.1","last_update":"2026-03-20T11:45:37.912792Z"},{"marker":"1_1774007157.850890_269.1","last_update":"2026-03-20T11:45:57.850890Z"},{"marker":"1_1774007039.336409_210.1","last_update":"2026-03-20T11:43:59.336409Z"}] 2026-03-20T11:45:58.370 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:45:58.370 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:58.370 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:45:58.370 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:45:58.446 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:58.446 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:58.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.521+0000 7f627c615900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:45:58.519 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.522+0000 7f627c615900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:45:58.528 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774007137.857782_141.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:45:37.857782Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774007137.912792_174.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:37.912792Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774007157.850890_269.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:57.850890Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774007039.336409_210.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:59.336409Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:45:58Z"} 2026-03-20T11:45:58.529 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:45:58.529 INFO:rgw_multi.tests:trimming on zone=a1 2026-03-20T11:45:58.529 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '50', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:45:58.529 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '50', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:45:58.529 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bilog autotrim --rgw-sync-log-trim-max-buckets 50 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 20 --debug-ms 0 2026-03-20T11:45:58.605 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:45:58.605 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:45:58.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.621+0000 7fa350c1d900 20 rados->read ofs=0 len=0 2026-03-20T11:45:58.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.622+0000 7fa350c1d900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:45:58.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.622+0000 7fa350c1d900 20 rados->read ofs=0 len=0 2026-03-20T11:45:58.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.623+0000 7fa350c1d900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:45:58.619 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.623+0000 7fa350c1d900 20 realm test-realm 6a789b89-c2b8-45f5-82cc-fb80234d0c6c 2026-03-20T11:45:58.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.623+0000 7fa350c1d900 20 rados->read ofs=0 len=0 2026-03-20T11:45:58.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.623+0000 7fa350c1d900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:45:58.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.623+0000 7fa350c1d900 20 rados->read ofs=0 len=0 2026-03-20T11:45:58.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.624+0000 7fa350c1d900 20 rados_obj.operate() r=0 bl.length=10 2026-03-20T11:45:58.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.624+0000 7fa350c1d900 20 rados->read ofs=0 len=0 2026-03-20T11:45:58.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.624+0000 7fa350c1d900 20 rados_obj.operate() r=0 bl.length=2020 2026-03-20T11:45:58.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.624+0000 7fa350c1d900 20 rados->read ofs=0 len=0 2026-03-20T11:45:58.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.624+0000 7fa350c1d900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:45:58.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.624+0000 7fa350c1d900 20 rados->read ofs=0 len=0 2026-03-20T11:45:58.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.625+0000 7fa350c1d900 20 rados_obj.operate() r=0 bl.length=1030 2026-03-20T11:45:58.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.625+0000 7fa350c1d900 20 zone a1 found 2026-03-20T11:45:58.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.625+0000 7fa350c1d900 4 Realm: test-realm (6a789b89-c2b8-45f5-82cc-fb80234d0c6c) 2026-03-20T11:45:58.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.625+0000 7fa350c1d900 4 ZoneGroup: a (a) 2026-03-20T11:45:58.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.625+0000 7fa350c1d900 4 Zone: a1 (4b634ebf-58f5-45bc-839a-cb4b425e773a) 2026-03-20T11:45:58.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.625+0000 7fa350c1d900 4 using period configuration: cb2fcdce-2c1b-429a-8601-a284ee3a2381:12 2026-03-20T11:45:58.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.625+0000 7fa350c1d900 20 period zonegroup name a 2026-03-20T11:45:58.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.625+0000 7fa350c1d900 20 using current period zonegroup a 2026-03-20T11:45:58.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.625+0000 7fa350c1d900 20 rados->read ofs=0 len=0 2026-03-20T11:45:58.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.625+0000 7fa350c1d900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.625+0000 7fa350c1d900 20 rados->read ofs=0 len=0 2026-03-20T11:45:58.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.626+0000 7fa350c1d900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.626+0000 7fa350c1d900 20 init(): pipe_map (zone=4b634ebf-58f5-45bc-839a-cb4b425e773a bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:45:58.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.626+0000 7fa350c1d900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.626+0000 7fa350c1d900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.626+0000 7fa350c1d900 20 rados->read ofs=0 len=0 2026-03-20T11:45:58.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.626+0000 7fa350c1d900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.626+0000 7fa350c1d900 20 rados->read ofs=0 len=0 2026-03-20T11:45:58.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.626+0000 7fa350c1d900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.626+0000 7fa350c1d900 20 init(): pipe_map (zone=51055576-6662-409d-8abb-ecaeefd3dd80 bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:45:58.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.626+0000 7fa350c1d900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.626+0000 7fa350c1d900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.626+0000 7fa350c1d900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.626+0000 7fa350c1d900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.626+0000 7fa350c1d900 20 started sync module instance, tier type = 2026-03-20T11:45:58.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.626+0000 7fa350c1d900 20 generating connection object for zone a2 id 51055576-6662-409d-8abb-ecaeefd3dd80 2026-03-20T11:45:58.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.626+0000 7fa350c1d900 20 started zone id=4b634ebf-58f5-45bc-839a-cb4b425e773a (name=a1) with tier type = 2026-03-20T11:45:58.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.629+0000 7fa350c1d900 20 add_watcher() i=4 2026-03-20T11:45:58.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.630+0000 7fa350c1d900 20 add_watcher() i=0 2026-03-20T11:45:58.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.630+0000 7fa350c1d900 20 add_watcher() i=2 2026-03-20T11:45:58.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.630+0000 7fa350c1d900 20 add_watcher() i=7 2026-03-20T11:45:58.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.631+0000 7fa350c1d900 20 add_watcher() i=6 2026-03-20T11:45:58.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.631+0000 7fa350c1d900 20 add_watcher() i=1 2026-03-20T11:45:58.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.631+0000 7fa350c1d900 20 add_watcher() i=5 2026-03-20T11:45:58.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.633+0000 7fa350c1d900 20 add_watcher() i=3 2026-03-20T11:45:58.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.633+0000 7fa350c1d900 2 all 8 watchers are set, enabling cache 2026-03-20T11:45:58.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.634+0000 7fa350c1d900 20 rgw_check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2026-03-20T11:45:58.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.634+0000 7fa350c1d900 20 rgw_check_secure_mon_conn(): mode 1 is insecure 2026-03-20T11:45:58.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.634+0000 7fa350c1d900 5 note: GC not initialized 2026-03-20T11:45:58.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.635+0000 7fa2f9fe3640 20 reqs_thread_entry: start 2026-03-20T11:45:58.688 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.691+0000 7fa350c1d900 20 init_complete bucket index max shards: 11 2026-03-20T11:45:58.688 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.691+0000 7fa350c1d900 20 Filter name: none 2026-03-20T11:45:58.688 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.691+0000 7fa2ab7fe640 20 reqs_thread_entry: start 2026-03-20T11:45:58.688 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.691+0000 7fa2a9ffb640 20 reqs_thread_entry: start 2026-03-20T11:45:58.690 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.693+0000 7fa350c1d900 10 rgw bucket trim manager: Watching bilog.trim 2026-03-20T11:45:58.690 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.693+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbc9f5e0:12BucketTrimCR: operate() 2026-03-20T11:45:58.690 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.693+0000 7fa350c1d900 10 fetching active bucket counters 2026-03-20T11:45:58.690 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.693+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca1d00:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:58.690 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.693+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca1d00:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:58.690 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.693+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is io blocked 2026-03-20T11:45:58.690 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.693+0000 7fa3237fe640 20 trim: get_bucket_counters: [] 2026-03-20T11:45:58.690 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.694+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca1d00:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:58.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.694+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca1d00:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:58.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.694+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca1d00:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:58.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.694+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca1d00:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:58.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.694+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbc9f5e0:12BucketTrimCR: operate() 2026-03-20T11:45:58.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.694+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca1340:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:58.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.694+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca1340:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:58.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.694+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is io blocked 2026-03-20T11:45:58.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.694+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca1340:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:58.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.694+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca1340:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:58.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.694+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca1340:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:58.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.694+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca1340:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:58.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.694+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbc9f5e0:12BucketTrimCR: operate() 2026-03-20T11:45:58.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.694+0000 7fa350c1d900 10 listing cold buckets from marker=12:b497a80d:root::.bucket.meta.dipepu-57%3a4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17:head 2026-03-20T11:45:58.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.694+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:14MetadataListCR: operate() 2026-03-20T11:45:58.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.694+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:14MetadataListCR: operate() 2026-03-20T11:45:58.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.694+0000 7fa350c1d900 20 enqueued request req=0x7fa308002c20 2026-03-20T11:45:58.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.694+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:58.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.694+0000 7fa350c1d900 20 req: 0x7fa308002c20 2026-03-20T11:45:58.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.694+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is io blocked 2026-03-20T11:45:58.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.694+0000 7fa2fb7e6640 20 dequeued request req=0x7fa308002c20 2026-03-20T11:45:58.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.694+0000 7fa2fb7e6640 20 RGWWQ: empty 2026-03-20T11:45:58.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.694+0000 7fa2fb7e6640 20 rgw async rados processor: starting metadata listing at 12:b497a80d:root::.bucket.meta.dipepu-57%3a4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17:head 2026-03-20T11:45:58.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.696+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:45:58.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.698+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-58 2026-03-20T11:45:58.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.698+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:45:58.696 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.700+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:45:58.698 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.701+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-27 2026-03-20T11:45:58.698 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.701+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-28 2026-03-20T11:45:58.698 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.701+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-59 2026-03-20T11:45:58.698 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.701+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:45:58.699 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.703+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:45:58.701 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.704+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-40 2026-03-20T11:45:58.701 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.704+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-22 2026-03-20T11:45:58.701 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.704+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:45:58.702 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.705+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-56 2026-03-20T11:45:58.702 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.705+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:45:58.703 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.706+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:45:58.704 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.708+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:45:58.705 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.708+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-53 2026-03-20T11:45:58.705 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.708+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:45:58.706 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.710+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:45:58.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.710+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:45:58.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.711+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:45:58.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.712+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:45:58.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.713+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-34 2026-03-20T11:45:58.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.713+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-21 2026-03-20T11:45:58.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.713+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-14 2026-03-20T11:45:58.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.713+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-50 2026-03-20T11:45:58.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.713+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-57 2026-03-20T11:45:58.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.713+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:45:58.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.713+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-39 2026-03-20T11:45:58.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.713+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:45:58.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.713+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-37 2026-03-20T11:45:58.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.713+0000 7fa2fb7e6640 20 rgw async rados processor: restarting metadata listing 2026-03-20T11:45:58.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.719+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-10 2026-03-20T11:45:58.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.719+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:45:58.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.725+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:45:58.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.731+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:45:58.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.736+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:45:58.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.742+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:45:58.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.747+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:45:58.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.752+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-19 2026-03-20T11:45:58.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.752+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-3 2026-03-20T11:45:58.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.752+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:45:58.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.758+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:45:58.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.763+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-17 2026-03-20T11:45:58.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.763+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:45:58.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.769+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-38 2026-03-20T11:45:58.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.769+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:45:58.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.774+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-24 2026-03-20T11:45:58.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.774+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-36 2026-03-20T11:45:58.771 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.774+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:45:58.775 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.779+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:45:58.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.784+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:45:58.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.788+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-44 2026-03-20T11:45:58.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.788+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-35 2026-03-20T11:45:58.785 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.788+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:45:58.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.792+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-16 2026-03-20T11:45:58.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.792+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:45:58.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.797+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:45:58.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.801+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:45:58.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.805+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-55 2026-03-20T11:45:58.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.805+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-13 2026-03-20T11:45:58.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.805+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-48 2026-03-20T11:45:58.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.805+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-25 2026-03-20T11:45:58.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.805+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-51 2026-03-20T11:45:58.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.805+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:45:58.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.809+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:45:58.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.813+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-46 2026-03-20T11:45:58.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.813+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:45:58.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.817+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:45:58.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.821+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:45:58.821 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.824+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-26 2026-03-20T11:45:58.821 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.824+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-20 2026-03-20T11:45:58.821 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.824+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:45:58.825 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.828+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:45:58.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.832+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-1 2026-03-20T11:45:58.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.832+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:45:58.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.835+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:45:58.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.839+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:45:58.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.842+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-52 2026-03-20T11:45:58.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.842+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-32 2026-03-20T11:45:58.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.842+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-42 2026-03-20T11:45:58.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.842+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:45:58.843 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.846+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:45:58.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.850+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-7 2026-03-20T11:45:58.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.850+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-54 2026-03-20T11:45:58.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.850+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-49 2026-03-20T11:45:58.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.850+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-47 2026-03-20T11:45:58.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.850+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:45:58.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.853+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-11 2026-03-20T11:45:58.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.853+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-12 2026-03-20T11:45:58.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.853+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-41 2026-03-20T11:45:58.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.853+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-23 2026-03-20T11:45:58.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.853+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-15 2026-03-20T11:45:58.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.853+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-45 2026-03-20T11:45:58.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.853+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:45:58.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.856+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:45:58.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.858+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-9 2026-03-20T11:45:58.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.858+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:45:58.860 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.860+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-29 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.860+0000 7fa2fb7e6640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.860+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:14MetadataListCR: operate() 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.860+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:14MetadataListCR: operate() 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.860+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:14MetadataListCR: operate() 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.860+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:14MetadataListCR: operate() 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.860+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbc9f5e0:12BucketTrimCR: operate() 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.860+0000 7fa350c1d900 4 collected 50 buckets for trim 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.860+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.860+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.860+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.860+0000 7fa350c1d900 4 starting trim on bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.860+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbca4290:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.860+0000 7fa350c1d900 4 starting trim on bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.860+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbca5560:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.860+0000 7fa350c1d900 4 starting trim on bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.860+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cbca6830:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.860+0000 7fa350c1d900 4 starting trim on bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.860+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca2610:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.860+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbca7b00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.860+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbca8250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.860+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cbca8c40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.860+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca2610:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.860+0000 7fa350c1d900 20 enqueued request req=0x5631cbca95f0 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.860+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.860+0000 7fa350c1d900 20 req: 0x5631cbca95f0 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is io blocked 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbca7b00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa350c1d900 20 enqueued request req=0x5631cb8cc640 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa350c1d900 20 req: 0x5631cbca95f0 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa350c1d900 20 req: 0x5631cb8cc640 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa350c1d900 20 run: stack=0x5631cbc2ed40 is io blocked 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbca8250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa350c1d900 20 enqueued request req=0x5631cb8cc9e0 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa350c1d900 20 req: 0x5631cbca95f0 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa350c1d900 20 req: 0x5631cb8cc640 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa350c1d900 20 req: 0x5631cb8cc9e0 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is io blocked 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cbca8c40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa350c1d900 20 enqueued request req=0x5631cb8cce00 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa350c1d900 20 req: 0x5631cbca95f0 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa350c1d900 20 req: 0x5631cb8cc640 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa350c1d900 20 req: 0x5631cb8cc9e0 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa350c1d900 20 req: 0x5631cb8cce00 2026-03-20T11:45:58.861 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa350c1d900 20 run: stack=0x5631cbca3750 is io blocked 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30e7fc640 20 dequeued request req=0x5631cbca95f0 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30e7fc640 20 RGWWQ: 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30e7fc640 20 req: 0x5631cb8cc640 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30e7fc640 20 req: 0x5631cb8cc9e0 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30e7fc640 20 req: 0x5631cb8cce00 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30e7fc640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 : miss 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30dffb640 20 dequeued request req=0x5631cb8cc640 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30dffb640 20 RGWWQ: 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30dffb640 20 req: 0x5631cb8cc9e0 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30dffb640 20 req: 0x5631cb8cce00 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30dffb640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 : miss 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30d7fa640 20 dequeued request req=0x5631cb8cc9e0 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30d7fa640 20 RGWWQ: 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30d7fa640 20 req: 0x5631cb8cce00 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30d7fa640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 : miss 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30effd640 20 dequeued request req=0x5631cb8cce00 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30effd640 20 RGWWQ: empty 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30effd640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 : miss 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30effd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30effd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30effd640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 info.flags=0x17 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30effd640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 to cache LRU end 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30effd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30effd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30effd640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30effd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30effd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-34 : miss 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30effd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30e7fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30e7fc640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 info.flags=0x17 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30e7fc640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 to cache LRU end 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30e7fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30e7fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30e7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30e7fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-57 : miss 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30e7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30e7fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-57 info.flags=0x0 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30e7fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-57 to cache LRU end 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30e7fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-57 : miss 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30effd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30effd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-34 info.flags=0x0 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30effd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-34 to cache LRU end 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30effd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-34 : miss 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.861+0000 7fa30effd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30e7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30e7fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-57 info.flags=0x0 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30e7fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-57 to cache LRU end 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17): adding source pipe: {s={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17): adding dest pipe: {s={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30e7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca2610:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca2610:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca2610:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.862 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca2610:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cb8cd220:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cb8cd840:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cb8cd840:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 15639b4287321268e81ee118624f47119da0347c11ad510bc50a8443e8f0b381 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114558Z 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:15639b4287321268e81ee118624f47119da0347c11ad510bc50a8443e8f0b381 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 1169405e7ff4c4c38ab6e85a2b80f7df357b64d13467f15f96375626e1a51fdc 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=1169405e7ff4c4c38ab6e85a2b80f7df357b64d13467f15f96375626e1a51fdc 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114558Z 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=0, curl_handle=0x5631cb8cf450 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is io blocked 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8cff10 req_data->id=0, curl_handle=0x5631cb8cf450 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30dffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30dffb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 info.flags=0x17 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30dffb640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 to cache LRU end 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30dffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30dffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30dffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30dffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-29 : miss 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30d7fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30d7fa640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 info.flags=0x17 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30d7fa640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 to cache LRU end 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30d7fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30d7fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30d7fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30d7fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-23 : miss 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30effd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.863 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30effd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-34 info.flags=0x0 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30effd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-34 to cache LRU end 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30effd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39): adding source pipe: {s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30effd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39): adding dest pipe: {s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30effd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30effd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30effd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cbca8c40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cbca8c40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cbca8c40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cbca8c40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cbca6830:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cb8d03c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cbca6830:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 run: stack=0x5631cbca3750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cb8bbe70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cb8bbe70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 1a3a227d50ab52a494b6297a943587c209c15d54851b9b8786cd0ab288f8f4ce 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114558Z 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:1a3a227d50ab52a494b6297a943587c209c15d54851b9b8786cd0ab288f8f4ce 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 5fc83ec24ef73a5bc8d81eec472c87d9cac3f35ddd15fd8fc8bc426695d33286 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5fc83ec24ef73a5bc8d81eec472c87d9cac3f35ddd15fd8fc8bc426695d33286 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114558Z 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=1, curl_handle=0x5631cb8bd9e0 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa350c1d900 20 run: stack=0x5631cbca9120 is io blocked 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8be0f0 req_data->id=1, curl_handle=0x5631cb8bd9e0 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30d7fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30d7fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-23 info.flags=0x0 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30d7fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-23 to cache LRU end 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30d7fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-23 : miss 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30dffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30dffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-29 info.flags=0x0 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30dffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-29 to cache LRU end 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30dffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-29 : miss 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.862+0000 7fa30dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.864 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa30d7fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa30d7fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-23 info.flags=0x0 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa30d7fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-23 to cache LRU end 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa30d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28): adding source pipe: {s={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa30d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28): adding dest pipe: {s={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa30d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa30d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa30d7fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbca8250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbca8250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbca8250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbca8250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbca5560:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cb8bf790:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbca5560:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cb8bfdb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cb8bfdb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 5f3ff95d251a05708dafc01ed5c46a6352a6e5fad907cb479705decc35377902 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114558Z 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:5f3ff95d251a05708dafc01ed5c46a6352a6e5fad907cb479705decc35377902 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 8870c9e0cfc69271f9ceb4d87688b29ae331c9402644c53be62e005469638f46 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=8870c9e0cfc69271f9ceb4d87688b29ae331c9402644c53be62e005469638f46 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114558Z 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=2, curl_handle=0x5631cb8c1990 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 run: stack=0x5631cbca8810 is io blocked 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8c20a0 req_data->id=2, curl_handle=0x5631cb8c1990 2026-03-20T11:45:58.865 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cb8bbe70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.866 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:58.862+0000 7fd05a302640 -1 req 15577877297421983849 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:58.866 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:58.864+0000 7fcfc81de640 -1 req 13438761512847133871 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cb8bbe70:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cb8d03c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 15 stack 0x5631cbca9120 end 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 run: stack=0x5631cbca9120 is done 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cbca6830:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 collect(): s=0x5631cbca3750 stack=0x5631cbca9120 is complete 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cbca6830:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 15 stack 0x5631cbca3750 end 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 stack->operate() returned ret=-22 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 run: stack=0x5631cbca3750 is done 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cbca6830:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 4 starting trim on bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cb8bbe70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cb8bbe70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 enqueued request req=0x5631cb8cce00 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 req: 0x5631cb8cce00 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa350c1d900 20 run: stack=0x5631cbca3750 is io blocked 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa30cff9640 20 dequeued request req=0x5631cb8cce00 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa30cff9640 20 RGWWQ: empty 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.863+0000 7fa30cff9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 : miss 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.864+0000 7fa30cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.864+0000 7fa30cff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.864+0000 7fa30cff9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 info.flags=0x17 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.864+0000 7fa30cff9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 to cache LRU end 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.864+0000 7fa30cff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.864+0000 7fa30cff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.864+0000 7fa30cff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.864+0000 7fa30cff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-54 : miss 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.864+0000 7fa30cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.864+0000 7fa30cff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.864+0000 7fa30cff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-54 info.flags=0x0 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.864+0000 7fa30cff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-54 to cache LRU end 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.864+0000 7fa30cff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-54 : miss 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.864+0000 7fa30cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa30dffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa30dffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-29 info.flags=0x0 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa30dffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-29 to cache LRU end 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa30dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34): adding source pipe: {s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa30dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34): adding dest pipe: {s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa30dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa30dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa30dffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbca7b00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbca7b00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbca7b00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbca7b00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbca4290:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cb8bc5c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbca4290:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 run: stack=0x5631cbc2ed40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cb8d03a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cb8d03a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = c9a42986e96ac4d288c0c3d4c6581febeb2d92f0db3e1550eea9dd6a4cc06686 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114558Z 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:c9a42986e96ac4d288c0c3d4c6581febeb2d92f0db3e1550eea9dd6a4cc06686 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = c445a6e95c77b333161dcd32b8852c4399e9a08eaa28c0602f674ea41ac6fff2 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c445a6e95c77b333161dcd32b8852c4399e9a08eaa28c0602f674ea41ac6fff2 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114558Z 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=3, curl_handle=0x5631cb8bd9e0 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 run: stack=0x5631cbca9120 is io blocked 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8be0f0 req_data->id=3, curl_handle=0x5631cb8bd9e0 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa30cff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa30cff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-54 info.flags=0x0 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa30cff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-54 to cache LRU end 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa30cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14): adding source pipe: {s={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa30cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14): adding dest pipe: {s={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa30cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa30cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa30cff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cb8bbe70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cb8bbe70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cb8bbe70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cb8bbe70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cbca6830:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 cr:s=0x5631cb8bc410:op=0x5631cbcab280:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cbca6830:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 run: stack=0x5631cbca3750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 cr:s=0x5631cb8bc410:op=0x5631cbcab8a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 cr:s=0x5631cb8bc410:op=0x5631cbcab8a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 16fc1fbf2a1aa6b09f63a935b604198692051eaa0dd410f9371f5741e15bfa1e 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114558Z 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:16fc1fbf2a1aa6b09f63a935b604198692051eaa0dd410f9371f5741e15bfa1e 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = cb8fe29c2e92ca7f005283fed9787976dd2a5c67af73f92d30962dacc7c4aca3 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=cb8fe29c2e92ca7f005283fed9787976dd2a5c67af73f92d30962dacc7c4aca3 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114558Z 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=4, curl_handle=0x5631cbcad4d0 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa350c1d900 20 run: stack=0x5631cb8bc410 is io blocked 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.865+0000 7fa2a9ffb640 20 link_request req_data=0x5631cbcadbe0 req_data->id=4, curl_handle=0x5631cbcad4d0 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cb8cd840:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cb8cd840:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cb8bfdb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cb8cd840:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cb8bfdb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cb8cd840:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cb8bfdb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cb8cd220:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 15 stack 0x5631cbca2ac0 end 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is done 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cb8bfdb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 collect(): s=0x5631cbd5d230 stack=0x5631cbca2ac0 is complete 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 10 trimming bilogs for bucket=:dipepu-57[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17]) markers=[,00000000001.41.6,,,,00000000001.23.6,,,,,], shards=11 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cb8bf790:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 15 stack 0x5631cbca8810 end 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 run: stack=0x5631cbca8810 is done 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbcaf9c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 10 trimming bilog shard 1 of :dipepu-57[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17]) at marker 00000000001.41.6 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 10 trimming bilog shard 5 of :dipepu-57[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17]) at marker 00000000001.23.6 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.869 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbca5560:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 collect(): s=0x5631cbc36db0 stack=0x5631cbca8810 is complete 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 10 trimming bilogs for bucket=:dipepu-23[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28]) markers=[,,,,,,,,00000000003.16.7,,], shards=11 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd76cc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbd71620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbcb00d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 10 trimming bilog shard 8 of :dipepu-23[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28]) at marker 00000000003.16.7 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd76cc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17.1 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is io blocked 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbd71620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17.5 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 run: stack=0x5631cbd79340 is io blocked 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cb8cd220:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cb8cd220:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28.8 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.866+0000 7fa350c1d900 20 run: stack=0x5631cbca8810 is io blocked 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 20 cr:s=0x5631cb8bc410:op=0x5631cbcab8a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 20 cr:s=0x5631cb8bc410:op=0x5631cbcab8a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 20 cr:s=0x5631cb8bc410:op=0x5631cbcab8a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 20 cr:s=0x5631cb8bc410:op=0x5631cbcab8a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 20 cr:s=0x5631cb8bc410:op=0x5631cbcab280:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 15 stack 0x5631cb8bc410 end 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 20 run: stack=0x5631cb8bc410 is done 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cbca6830:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 20 collect(): s=0x5631cbca3750 stack=0x5631cb8bc410 is complete 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 10 trimming bilogs for bucket=:dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) markers=[00000000004.46.6,00000000002.37.1,00000000004.60.6,00000000002.19.1,00000000004.18.6,00000000002.57.1,00000000002.32.1,00000000004.68.6,00000000002.43.1,00000000002.85.1,00000000002.30.1], shards=11 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cbd78cc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 10 trimming bilog shard 0 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.46.6 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 10 trimming bilog shard 1 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.37.1 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 10 trimming bilog shard 2 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.60.6 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 10 trimming bilog shard 3 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.19.1 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 10 trimming bilog shard 4 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.18.6 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 10 trimming bilog shard 5 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.57.1 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 10 trimming bilog shard 6 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.32.1 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 10 trimming bilog shard 7 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.68.6 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 10 trimming bilog shard 8 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.43.1 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 10 trimming bilog shard 9 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.85.1 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 10 trimming bilog shard 10 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.30.1 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 20 run: stack=0x5631cbca3750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 20 cr:s=0x5631cb8bc410:op=0x5631cbd73410:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 20 cr:s=0x5631cbd80510:op=0x5631cbd7da10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbd84490:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 20 cr:s=0x5631cbcb7980:op=0x5631cbd837a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbcb17d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 20 cr:s=0x5631cbd7b010:op=0x5631cbd7bc30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 20 cr:s=0x5631cbcb6930:op=0x5631cbd79950:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 20 cr:s=0x5631cbd80360:op=0x5631cbd81ac0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd82960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 20 cr:s=0x5631cbd7eb90:op=0x5631cbcb5970:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 20 cr:s=0x5631cbd89190:op=0x5631cbd72100:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 20 cr:s=0x5631cb8bc410:op=0x5631cbd73410:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.0 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 20 run: stack=0x5631cb8bc410 is io blocked 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 20 cr:s=0x5631cbd80510:op=0x5631cbd7da10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.870 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.867+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.1 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 run: stack=0x5631cbd80510 is io blocked 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbd84490:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.2 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 run: stack=0x5631cbcb7f60 is io blocked 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 cr:s=0x5631cbcb7980:op=0x5631cbd837a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.3 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 run: stack=0x5631cbcb7980 is io blocked 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbcb17d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.4 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is io blocked 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 cr:s=0x5631cbd7b010:op=0x5631cbd7bc30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.5 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 run: stack=0x5631cbd7b010 is io blocked 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 cr:s=0x5631cbcb6930:op=0x5631cbd79950:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.6 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 run: stack=0x5631cbcb6930 is io blocked 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 cr:s=0x5631cbd80360:op=0x5631cbd81ac0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.7 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 run: stack=0x5631cbd80360 is io blocked 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd82960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.8 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 run: stack=0x5631cbcb4470 is io blocked 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 cr:s=0x5631cbd7eb90:op=0x5631cbcb5970:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.9 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 run: stack=0x5631cbd7eb90 is io blocked 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 cr:s=0x5631cbd89190:op=0x5631cbd72100:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.10 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 run: stack=0x5631cbd89190 is io blocked 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd76cc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd76cc0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 15 stack 0x5631cbca2ac0 end 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is done 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbcaf9c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.868+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbd71620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbd71620:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 15 stack 0x5631cbd79340 end 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 run: stack=0x5631cbd79340 is done 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbcaf9c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 trim: trimmed bucket instance dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 15 stack 0x5631cbd5d230 end 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is done 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 4 starting trim on bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbd71620:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbd71620:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 enqueued request req=0x5631cbca3980 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 req: 0x5631cbca3980 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is io blocked 2026-03-20T11:45:58.871 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cb8cd220:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.872 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:58.871+0000 7fd06631a640 -1 req 4000919271463128559 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:58.872 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:58.871+0000 7fd088b5f640 -1 req 6710342555160226243 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:58.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cb8cd220:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 15 stack 0x5631cbca8810 end 2026-03-20T11:45:58.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 run: stack=0x5631cbca8810 is done 2026-03-20T11:45:58.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbcb00d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbca5560:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 trim: trimmed bucket instance dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:45:58.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 15 stack 0x5631cbc36db0 end 2026-03-20T11:45:58.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is done 2026-03-20T11:45:58.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:58.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbca5560:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 4 starting trim on bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:45:58.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd76ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd76ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 enqueued request req=0x5631cb8cf980 2026-03-20T11:45:58.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:58.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 req: 0x5631cbca3980 2026-03-20T11:45:58.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 req: 0x5631cb8cf980 2026-03-20T11:45:58.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is io blocked 2026-03-20T11:45:58.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 cr:s=0x5631cbd80510:op=0x5631cbd7da10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 cr:s=0x5631cbd80510:op=0x5631cbd7da10:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 15 stack 0x5631cbd80510 end 2026-03-20T11:45:58.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 run: stack=0x5631cbd80510 is done 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cbd78cc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.869+0000 7fa350c1d900 20 run: stack=0x5631cbca3750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbcb17d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbcb17d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 15 stack 0x5631cbcb6b00 end 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is done 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cbd78cc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 run: stack=0x5631cbca3750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa307fff640 20 dequeued request req=0x5631cbca3980 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa307fff640 20 RGWWQ: 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa307fff640 20 req: 0x5631cb8cf980 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa307fff640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 : miss 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa307fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbd84490:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbd84490:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 15 stack 0x5631cbcb7f60 end 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 run: stack=0x5631cbcb7f60 is done 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cbd78cc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa3077fe640 20 dequeued request req=0x5631cb8cf980 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 run: stack=0x5631cbca3750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa3077fe640 20 RGWWQ: empty 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa3077fe640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 : miss 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa3077fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd82960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd82960:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 15 stack 0x5631cbcb4470 end 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 run: stack=0x5631cbcb4470 is done 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cbd78cc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 run: stack=0x5631cbca3750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cb8bc410:op=0x5631cbd73410:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cb8bc410:op=0x5631cbd73410:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 15 stack 0x5631cb8bc410 end 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 run: stack=0x5631cb8bc410 is done 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cbd78cc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 run: stack=0x5631cbca3750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbd7eb90:op=0x5631cbcb5970:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbd7eb90:op=0x5631cbcb5970:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 15 stack 0x5631cbd7eb90 end 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 run: stack=0x5631cbd7eb90 is done 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cbd78cc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 run: stack=0x5631cbca3750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa307fff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbcb7980:op=0x5631cbd837a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbcb7980:op=0x5631cbd837a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 15 stack 0x5631cbcb7980 end 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 run: stack=0x5631cbcb7980 is done 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa307fff640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 info.flags=0x17 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cbd78cc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 run: stack=0x5631cbca3750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa307fff640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 to cache LRU end 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa307fff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa307fff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa307fff640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa307fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa307fff640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-32 : miss 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa307fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbd7b010:op=0x5631cbd7bc30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa3077fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbd7b010:op=0x5631cbd7bc30:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 15 stack 0x5631cbd7b010 end 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 run: stack=0x5631cbd7b010 is done 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa3077fe640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 info.flags=0x17 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cbd78cc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa3077fe640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 to cache LRU end 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 run: stack=0x5631cbca3750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa3077fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa3077fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbcb6930:op=0x5631cbd79950:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbcb6930:op=0x5631cbd79950:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 15 stack 0x5631cbcb6930 end 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 run: stack=0x5631cbcb6930 is done 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbd89190:op=0x5631cbd72100:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbd89190:op=0x5631cbd72100:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 15 stack 0x5631cbd89190 end 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa3077fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 run: stack=0x5631cbd89190 is done 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbd80360:op=0x5631cbd81ac0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbd80360:op=0x5631cbd81ac0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 15 stack 0x5631cbd80360 end 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 run: stack=0x5631cbd80360 is done 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cbd78cc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cbd78cc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cbd78cc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbca3750:op=0x5631cbca6830:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 trim: trimmed bucket instance dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 15 stack 0x5631cbca3750 end 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa3077fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-19 : miss 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 run: stack=0x5631cbca3750 is done 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa3077fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbca6830:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 4 starting trim on bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd73410:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd73410:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 enqueued request req=0x5631cb8cce00 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 req: 0x5631cb8cce00 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa350c1d900 20 run: stack=0x5631cbcb4470 is io blocked 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa306ffd640 20 dequeued request req=0x5631cb8cce00 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa306ffd640 20 RGWWQ: empty 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa306ffd640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 : miss 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa306ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa307fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa307fff640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-32 info.flags=0x0 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa307fff640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-32 to cache LRU end 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa307fff640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-32 : miss 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.870+0000 7fa307fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa3077fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa3077fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-19 info.flags=0x0 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa3077fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-19 to cache LRU end 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa3077fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-19 : miss 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa3077fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa306ffd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa306ffd640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 info.flags=0x17 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa306ffd640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 to cache LRU end 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa306ffd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:58.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa306ffd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa306ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa306ffd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-4 : miss 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa306ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa307fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa307fff640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-32 info.flags=0x0 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa307fff640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-32 to cache LRU end 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa307fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37): adding source pipe: {s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa307fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37): adding dest pipe: {s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa307fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa307fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa307fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbd71620:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbd71620:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbd71620:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbd71620:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbd80d50:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbcb6c40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbcb6c40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa306ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa306ffd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-4 info.flags=0x0 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa306ffd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-4 to cache LRU end 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa306ffd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-4 : miss 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa306ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa3077fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa3077fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-19 info.flags=0x0 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa3077fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-19 to cache LRU end 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa3077fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24): adding source pipe: {s={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa3077fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24): adding dest pipe: {s={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa3077fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa3077fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 2974b46e594b4b685448beb152149d30d0e692d43912ea4157fd33a1439cbe40 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa3077fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114558Z 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2974b46e594b4b685448beb152149d30d0e692d43912ea4157fd33a1439cbe40 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = df602e139e9bb2d720128bc2d1fc0b5b196e56366f9e673cfe3077295bfb2031 2026-03-20T11:45:58.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=df602e139e9bb2d720128bc2d1fc0b5b196e56366f9e673cfe3077295bfb2031 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114558Z 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=5, curl_handle=0x5631cbcad4d0 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 run: stack=0x5631cbcb7f60 is io blocked 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd76ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd76ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd76ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd76ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbca5560:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa2a9ffb640 20 link_request req_data=0x5631cbcadbe0 req_data->id=5, curl_handle=0x5631cbcad4d0 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd76ca0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbca5560:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd71620:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd71620:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = ced6b8cbdff699f9483c37586e9874173684f37132a34e99a741f8ae707b601c 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114558Z 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:ced6b8cbdff699f9483c37586e9874173684f37132a34e99a741f8ae707b601c 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = d272bba4790ec5872fa45b17babcffaae2783b5f034041af696af746d41bb2ff 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=d272bba4790ec5872fa45b17babcffaae2783b5f034041af696af746d41bb2ff 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114558Z 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=6, curl_handle=0x5631cb8c1990 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is io blocked 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8c20a0 req_data->id=6, curl_handle=0x5631cb8c1990 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa306ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa306ffd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-4 info.flags=0x0 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa306ffd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-4 to cache LRU end 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa306ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5): adding source pipe: {s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa306ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5): adding dest pipe: {s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa306ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa306ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa306ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd73410:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd73410:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd73410:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd73410:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbca6830:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 cr:s=0x5631cbd80510:op=0x5631cbd73410:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbca6830:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 run: stack=0x5631cbcb4470 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 cr:s=0x5631cbd80510:op=0x5631cbd79950:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 cr:s=0x5631cbd80510:op=0x5631cbd79950:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = a29126291ffe3922b633dff2e09d063ba09785b68e19a8188fed4137934a9a40 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114558Z 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:a29126291ffe3922b633dff2e09d063ba09785b68e19a8188fed4137934a9a40 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 0afc288c617c707005b5f1cc9cc24389dfd66acf6355e8b06707e9704d3854ea 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=0afc288c617c707005b5f1cc9cc24389dfd66acf6355e8b06707e9704d3854ea 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114558Z 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=7, curl_handle=0x5631cb8cf450 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa350c1d900 20 run: stack=0x5631cbd80510 is io blocked 2026-03-20T11:45:58.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.871+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8cff10 req_data->id=7, curl_handle=0x5631cb8cf450 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cb8d03a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cb8d03a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cb8bc5c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa350c1d900 15 stack 0x5631cbca9120 end 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa350c1d900 20 run: stack=0x5631cbca9120 is done 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbca4290:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa350c1d900 20 collect(): s=0x5631cbc2ed40 stack=0x5631cbca9120 is complete 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa350c1d900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa350c1d900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbca4290:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa350c1d900 15 stack 0x5631cbc2ed40 end 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa350c1d900 20 stack->operate() returned ret=-22 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa350c1d900 20 run: stack=0x5631cbc2ed40 is done 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa350c1d900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbca4290:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa350c1d900 4 starting trim on bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbd79f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbd79f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa350c1d900 20 enqueued request req=0x5631cb8cc640 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa350c1d900 20 req: 0x5631cb8cc640 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa350c1d900 20 run: stack=0x5631cbc2ed40 is io blocked 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa3067fc640 20 dequeued request req=0x5631cb8cc640 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa3067fc640 20 RGWWQ: empty 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa3067fc640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 : miss 2026-03-20T11:45:58.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa3067fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa3067fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa3067fc640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 info.flags=0x17 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa3067fc640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 to cache LRU end 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa3067fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa3067fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa3067fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa3067fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-53 : miss 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.906+0000 7fa3067fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa3067fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa3067fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-53 info.flags=0x0 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa3067fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-53 to cache LRU end 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa3067fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-53 : miss 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa3067fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa3067fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa3067fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-53 info.flags=0x0 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa3067fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-53 to cache LRU end 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa3067fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13): adding source pipe: {s={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa3067fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13): adding dest pipe: {s={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa3067fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa3067fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa3067fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbd79f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbd79f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbd79f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbd79f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbca4290:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cb8bc5c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbca4290:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 20 run: stack=0x5631cbc2ed40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cb8d03a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cb8d03a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:58.904 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.905 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.905 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.905 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.905 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.905 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:58.905 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 07c0bc3a5a13485271f033561850f31add39d8d32bfaca432a4259c60c4aaf08 2026-03-20T11:45:58.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:58.905 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114558Z 2026-03-20T11:45:58.905 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:58.905 INFO:teuthology.orchestra.run.vm02.stderr:07c0bc3a5a13485271f033561850f31add39d8d32bfaca432a4259c60c4aaf08 2026-03-20T11:45:58.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:58.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:58.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:58.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:58.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 889791100c364fe1c109ae49e0be1994ba34a94b6cf506d04d830edbf6e8fed0 2026-03-20T11:45:58.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=889791100c364fe1c109ae49e0be1994ba34a94b6cf506d04d830edbf6e8fed0 2026-03-20T11:45:58.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114558Z 2026-03-20T11:45:58.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:58.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=8, curl_handle=0x5631cb8bd9e0 2026-03-20T11:45:58.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa350c1d900 20 run: stack=0x5631cbca9120 is io blocked 2026-03-20T11:45:58.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.907+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8be0f0 req_data->id=8, curl_handle=0x5631cb8bd9e0 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbcb6c40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbcb6c40:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbd80d50:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 15 stack 0x5631cbcb7f60 end 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 run: stack=0x5631cbcb7f60 is done 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 cr:s=0x5631cbd80510:op=0x5631cbd79950:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 cr:s=0x5631cbd80510:op=0x5631cbd79950:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 collect(): s=0x5631cbd5d230 stack=0x5631cbcb7f60 is complete 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 15 stack 0x5631cbd5d230 end 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 stack->operate() returned ret=-22 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is done 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 cr:s=0x5631cbd80510:op=0x5631cbd73410:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 15 stack 0x5631cbd80510 end 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 run: stack=0x5631cbd80510 is done 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbca6830:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 collect(): s=0x5631cbcb4470 stack=0x5631cbd80510 is complete 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbca6830:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 15 stack 0x5631cbcb4470 end 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 stack->operate() returned ret=-22 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 run: stack=0x5631cbcb4470 is done 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 4 starting trim on bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbd73410:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbca6830:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 4 starting trim on bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbd73410:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 enqueued request req=0x5631cbcb7510 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 req: 0x5631cbcb7510 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is io blocked 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd7e4a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd7e4a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 enqueued request req=0x5631cbcaf670 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 req: 0x5631cbcb7510 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 req: 0x5631cbcaf670 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa350c1d900 20 run: stack=0x5631cbcb4470 is io blocked 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa305ffb640 20 dequeued request req=0x5631cbcb7510 2026-03-20T11:45:58.910 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa305ffb640 20 RGWWQ: 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa305ffb640 20 req: 0x5631cbcaf670 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa305ffb640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 : miss 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa305ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa3057fa640 20 dequeued request req=0x5631cbcaf670 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa3057fa640 20 RGWWQ: empty 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa3057fa640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 : miss 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa3057fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa305ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa305ffb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 info.flags=0x17 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa305ffb640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 to cache LRU end 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa305ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa305ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa305ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa305ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-14 : miss 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa305ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa3057fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa3057fa640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 info.flags=0x17 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa3057fa640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 to cache LRU end 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa3057fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa3057fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa3057fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa3057fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-36 : miss 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa3057fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa305ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa305ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-14 info.flags=0x0 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa305ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-14 to cache LRU end 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa305ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-14 : miss 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.913+0000 7fa305ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa3057fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa3057fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-36 info.flags=0x0 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa3057fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-36 to cache LRU end 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa3057fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-36 : miss 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa3057fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd71620:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa305ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa305ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-14 info.flags=0x0 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa305ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-14 to cache LRU end 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa305ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19): adding source pipe: {s={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa305ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19): adding dest pipe: {s={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa305ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa305ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa305ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd71620:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbd73410:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd71620:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbd73410:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd71620:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbd73410:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd76ca0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 15 stack 0x5631cbcb6b00 end 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is done 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbd73410:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbca5560:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 collect(): s=0x5631cbc36db0 stack=0x5631cbcb6b00 is complete 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 10 trimming bilogs for bucket=:dipepu-19[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24]) markers=[,,,,,,,,00000000003.27.6,,], shards=11 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbcb00d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 10 trimming bilog shard 8 of :dipepu-19[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24]) at marker 00000000003.27.6 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbcb6c40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 cr:s=0x5631cbd80510:op=0x5631cbd76ca0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd7f450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 cr:s=0x5631cbd80510:op=0x5631cbd76ca0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24.8 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 run: stack=0x5631cbd80510 is io blocked 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd7f450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 44e8443eae47dac847e5186a9192f7a6fea83deaa96112f87c1f90edc1183bf3 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114558Z 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:44e8443eae47dac847e5186a9192f7a6fea83deaa96112f87c1f90edc1183bf3 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 9a1b209b4e11fc753b083f7a1cc009cf0250bb35989e320981ef0ab6eb9100e2 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=9a1b209b4e11fc753b083f7a1cc009cf0250bb35989e320981ef0ab6eb9100e2 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114558Z 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:58.912 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=9, curl_handle=0x5631cb8c1990 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is io blocked 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8c20a0 req_data->id=9, curl_handle=0x5631cb8c1990 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa3057fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa3057fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-36 info.flags=0x0 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa3057fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-36 to cache LRU end 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa3057fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41): adding source pipe: {s={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa3057fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41): adding dest pipe: {s={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa3057fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa3057fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa3057fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd7e4a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd7e4a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd7e4a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd7e4a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.914+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbca6830:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.915+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbd74f70:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.915+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbca6830:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.915+0000 7fa350c1d900 20 run: stack=0x5631cbcb4470 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.915+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cb8ce9e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.915+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cb8ce9e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.915+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.915+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.915+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.915+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.915+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.915+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.915+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = f21e8a2e3c4df3364cf60981d16ae3a505b47e64704456f7f968fc0f95fd813b 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.915+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114558Z 2026-03-20T11:45:58.913 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:f21e8a2e3c4df3364cf60981d16ae3a505b47e64704456f7f968fc0f95fd813b 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.915+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.915+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.915+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.915+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.915+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 3728a214c30c8d73dbf9a021b2a3f02925576c342b745481e991e67810227203 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.915+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=3728a214c30c8d73dbf9a021b2a3f02925576c342b745481e991e67810227203 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.915+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.915+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114558Z 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.915+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.915+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=10, curl_handle=0x5631cb8cf450 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.915+0000 7fa350c1d900 20 run: stack=0x5631cbcb7f60 is io blocked 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.915+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8cff10 req_data->id=10, curl_handle=0x5631cb8cf450 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa350c1d900 20 cr:s=0x5631cbd80510:op=0x5631cbd76ca0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa350c1d900 20 cr:s=0x5631cbd80510:op=0x5631cbd76ca0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa350c1d900 15 stack 0x5631cbd80510 end 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa350c1d900 20 run: stack=0x5631cbd80510 is done 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbcb00d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbca5560:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa350c1d900 20 trim: trimmed bucket instance dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa350c1d900 15 stack 0x5631cbc36db0 end 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is done 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbca5560:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa350c1d900 4 starting trim on bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd76ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd76ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa350c1d900 20 enqueued request req=0x5631cbca95f0 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa350c1d900 20 req: 0x5631cbca95f0 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is io blocked 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa304ff9640 20 dequeued request req=0x5631cbca95f0 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa304ff9640 20 RGWWQ: empty 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa304ff9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 : miss 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa304ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa304ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:58.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa304ff9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 info.flags=0x17 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa304ff9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 to cache LRU end 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa304ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa304ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa304ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa304ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-22 : miss 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa304ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa304ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa304ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-22 info.flags=0x0 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa304ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-22 to cache LRU end 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa304ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-22 : miss 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.916+0000 7fa304ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa304ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa304ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-22 info.flags=0x0 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa304ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-22 to cache LRU end 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa304ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27): adding source pipe: {s={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa304ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27): adding dest pipe: {s={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa304ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa304ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa304ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd76ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd76ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd76ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd76ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbca5560:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 20 cr:s=0x5631cbd80510:op=0x5631cbd80c90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbca5560:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 20 cr:s=0x5631cbd80510:op=0x5631cbd76ca0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 20 cr:s=0x5631cbd80510:op=0x5631cbd76ca0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:58 +0000 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114558Z 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 3087e1a4a1c37e9a89123f6d006d923477f8227706e3a2277e8cbe17e55685f3 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114558Z 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:3087e1a4a1c37e9a89123f6d006d923477f8227706e3a2277e8cbe17e55685f3 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 606609b414f03e7b09f1f45b294525ac8c68c5f9115fcfe00a80827ff68ee428 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=606609b414f03e7b09f1f45b294525ac8c68c5f9115fcfe00a80827ff68ee428 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114558Z 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:58.915 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=11, curl_handle=0x5631cbcad4d0 2026-03-20T11:45:58.916 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa350c1d900 20 run: stack=0x5631cbd80510 is io blocked 2026-03-20T11:45:58.916 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.917+0000 7fa2a9ffb640 20 link_request req_data=0x5631cbcadbe0 req_data->id=11, curl_handle=0x5631cbcad4d0 2026-03-20T11:45:58.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cb8d03a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cb8d03a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cb8d03a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cb8d03a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cb8bc5c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 15 stack 0x5631cbca9120 end 2026-03-20T11:45:58.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 run: stack=0x5631cbca9120 is done 2026-03-20T11:45:58.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbca4290:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 collect(): s=0x5631cbc2ed40 stack=0x5631cbca9120 is complete 2026-03-20T11:45:58.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:58.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 10 trimming bilogs for bucket=:dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) markers=[00000000004.65.1,00000000002.83.1,00000000004.51.1,00000000002.21.1,00000000004.74.1,00000000002.72.1,00000000002.84.1,00000000004.73.1,00000000002.22.1,00000000002.50.1,00000000002.35.1], shards=11 2026-03-20T11:45:58.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbcaf9c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 10 trimming bilog shard 0 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.65.1 2026-03-20T11:45:58.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 10 trimming bilog shard 1 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.83.1 2026-03-20T11:45:58.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 10 trimming bilog shard 2 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.51.1 2026-03-20T11:45:58.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 10 trimming bilog shard 3 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.21.1 2026-03-20T11:45:58.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 10 trimming bilog shard 4 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.74.1 2026-03-20T11:45:58.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 10 trimming bilog shard 5 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.72.1 2026-03-20T11:45:58.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 10 trimming bilog shard 6 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.84.1 2026-03-20T11:45:58.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 10 trimming bilog shard 7 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.73.1 2026-03-20T11:45:58.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 10 trimming bilog shard 8 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.22.1 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 10 trimming bilog shard 9 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.50.1 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 10 trimming bilog shard 10 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.35.1 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 run: stack=0x5631cbc2ed40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cb8c2440:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd84400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbd837a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd7ca00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cbd88de0:op=0x5631cbd7bea0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cb8c2c40:op=0x5631cbcb2c00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbd71620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cbd7b300:op=0x5631cbd72300:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbd7d4e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbd871c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cbd7af80:op=0x5631cbcb14c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cb8c2440:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.0 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 run: stack=0x5631cbca9120 is io blocked 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd84400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.1 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 run: stack=0x5631cbca8810 is io blocked 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbd837a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.2 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 run: stack=0x5631cbd79340 is io blocked 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd7ca00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.3 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is io blocked 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cbd88de0:op=0x5631cbd7bea0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.4 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 run: stack=0x5631cbd88de0 is io blocked 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cb8c2c40:op=0x5631cbcb2c00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.5 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 run: stack=0x5631cb8c2c40 is io blocked 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbd71620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.6 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 run: stack=0x5631cbcabf70 is io blocked 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cbd7b300:op=0x5631cbd72300:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.7 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 run: stack=0x5631cbd7b300 is io blocked 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbd7d4e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.8 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 run: stack=0x5631cbcb3810 is io blocked 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbd871c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.9 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is io blocked 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 cr:s=0x5631cbd7af80:op=0x5631cbcb14c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.10 2026-03-20T11:45:58.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.950+0000 7fa350c1d900 20 run: stack=0x5631cbd7af80 is io blocked 2026-03-20T11:45:58.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd7f450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.954 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd7f450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.954 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cb8ce9e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.954 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd7f450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cb8ce9e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd7f450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cb8ce9e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbcb6c40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 15 stack 0x5631cbcb6b00 end 2026-03-20T11:45:58.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is done 2026-03-20T11:45:58.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cb8ce9e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 collect(): s=0x5631cbd5d230 stack=0x5631cbcb6b00 is complete 2026-03-20T11:45:58.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:58.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 10 trimming bilogs for bucket=:dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) markers=[,,,,,00000000002.32.11,,,00000000003.22.6,00000000011.52.6,], shards=11 2026-03-20T11:45:58.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbd74f70:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 15 stack 0x5631cbcb7f60 end 2026-03-20T11:45:58.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 run: stack=0x5631cbcb7f60 is done 2026-03-20T11:45:58.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbda7d60:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 10 trimming bilog shard 5 of :dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) at marker 00000000002.32.11 2026-03-20T11:45:58.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 10 trimming bilog shard 8 of :dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) at marker 00000000003.22.6 2026-03-20T11:45:58.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 10 trimming bilog shard 9 of :dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) at marker 00000000011.52.6 2026-03-20T11:45:58.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbca6830:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 collect(): s=0x5631cbcb4470 stack=0x5631cbcb7f60 is complete 2026-03-20T11:45:58.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:58.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 10 trimming bilogs for bucket=:dipepu-36[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41]) markers=[,,,,,,,,,,00000000001.34.6], shards=11 2026-03-20T11:45:58.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd79fc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 cr:s=0x5631cbd75bc0:op=0x5631cbda67a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 cr:s=0x5631cb8c1740:op=0x5631cbd7e4a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.957 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd74f70:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 10 trimming bilog shard 10 of :dipepu-36[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41]) at marker 00000000001.34.6 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 run: stack=0x5631cbcb4470 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd79fc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19.5 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is io blocked 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 cr:s=0x5631cbd75bc0:op=0x5631cbda67a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19.8 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 run: stack=0x5631cbd75bc0 is io blocked 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 cr:s=0x5631cb8c1740:op=0x5631cbd7e4a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19.9 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 run: stack=0x5631cb8c1740 is io blocked 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cb8bfd90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cb8bfd90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41.10 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.957+0000 7fa350c1d900 20 run: stack=0x5631cbcb7f60 is io blocked 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.959+0000 7fa350c1d900 20 cr:s=0x5631cbd80510:op=0x5631cbd76ca0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.959+0000 7fa350c1d900 20 cr:s=0x5631cbd80510:op=0x5631cbd76ca0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.959+0000 7fa350c1d900 20 cr:s=0x5631cbd80510:op=0x5631cbd76ca0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.959+0000 7fa350c1d900 20 cr:s=0x5631cbd80510:op=0x5631cbd76ca0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.959+0000 7fa350c1d900 20 cr:s=0x5631cbd80510:op=0x5631cbd80c90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.959+0000 7fa350c1d900 15 stack 0x5631cbd80510 end 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.959+0000 7fa350c1d900 20 run: stack=0x5631cbd80510 is done 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.959+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbca5560:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.959+0000 7fa350c1d900 20 collect(): s=0x5631cbc36db0 stack=0x5631cbd80510 is complete 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.959+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.959+0000 7fa350c1d900 10 trimming bilogs for bucket=:dipepu-22[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27]) markers=[,,,,,,,,00000000006.25.9,,], shards=11 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.959+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbcb6c40:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.959+0000 7fa350c1d900 10 trimming bilog shard 8 of :dipepu-22[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27]) at marker 00000000006.25.9 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.959+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.959+0000 7fa350c1d900 20 cr:s=0x5631cbd80510:op=0x5631cbd76ca0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.959+0000 7fa350c1d900 20 cr:s=0x5631cbd80510:op=0x5631cbd76ca0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.959+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27.8 2026-03-20T11:45:58.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.959+0000 7fa350c1d900 20 run: stack=0x5631cbd80510 is io blocked 2026-03-20T11:45:58.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.968+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbd871c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.968+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbd871c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.968+0000 7fa350c1d900 15 stack 0x5631cbd84c00 end 2026-03-20T11:45:58.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.968+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.968+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is done 2026-03-20T11:45:58.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.968+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbcaf9c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.968+0000 7fa350c1d900 20 run: stack=0x5631cbc2ed40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.968+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbd837a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.968+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbd837a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.968+0000 7fa350c1d900 15 stack 0x5631cbd79340 end 2026-03-20T11:45:58.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.968+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.968+0000 7fa350c1d900 20 run: stack=0x5631cbd79340 is done 2026-03-20T11:45:58.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.968+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbcaf9c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.968+0000 7fa350c1d900 20 run: stack=0x5631cbc2ed40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.976+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cb8c2440:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.976+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cb8c2440:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.976+0000 7fa350c1d900 15 stack 0x5631cbca9120 end 2026-03-20T11:45:58.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.976+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.976+0000 7fa350c1d900 20 run: stack=0x5631cbca9120 is done 2026-03-20T11:45:58.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.976+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbcaf9c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.976+0000 7fa350c1d900 20 run: stack=0x5631cbc2ed40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.982+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd84400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.982+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd84400:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.982+0000 7fa350c1d900 15 stack 0x5631cbca8810 end 2026-03-20T11:45:58.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.982+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.982+0000 7fa350c1d900 20 run: stack=0x5631cbca8810 is done 2026-03-20T11:45:58.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.982+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbcaf9c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.982+0000 7fa350c1d900 20 run: stack=0x5631cbc2ed40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.982+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd7ca00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.982+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd7ca00:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.982+0000 7fa350c1d900 15 stack 0x5631cbca2ac0 end 2026-03-20T11:45:58.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.982+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.982+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is done 2026-03-20T11:45:58.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.982+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbcaf9c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.982+0000 7fa350c1d900 20 run: stack=0x5631cbc2ed40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.982+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbd71620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.982+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbd71620:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.982+0000 7fa350c1d900 15 stack 0x5631cbcabf70 end 2026-03-20T11:45:58.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.982+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.982+0000 7fa350c1d900 20 run: stack=0x5631cbcabf70 is done 2026-03-20T11:45:58.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.982+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbcaf9c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.982+0000 7fa350c1d900 20 run: stack=0x5631cbc2ed40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.982+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbd7d4e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.982+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbd7d4e0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.982+0000 7fa350c1d900 15 stack 0x5631cbcb3810 end 2026-03-20T11:45:58.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.982+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.982+0000 7fa350c1d900 20 run: stack=0x5631cbcb3810 is done 2026-03-20T11:45:58.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.982+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbcaf9c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.982+0000 7fa350c1d900 20 run: stack=0x5631cbc2ed40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.989+0000 7fa350c1d900 20 cr:s=0x5631cbd7af80:op=0x5631cbcb14c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.989+0000 7fa350c1d900 20 cr:s=0x5631cbd7af80:op=0x5631cbcb14c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.989+0000 7fa350c1d900 15 stack 0x5631cbd7af80 end 2026-03-20T11:45:58.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.989+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.989+0000 7fa350c1d900 20 run: stack=0x5631cbd7af80 is done 2026-03-20T11:45:58.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.989+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbcaf9c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.989+0000 7fa350c1d900 20 run: stack=0x5631cbc2ed40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.996+0000 7fa350c1d900 20 cr:s=0x5631cb8c2c40:op=0x5631cbcb2c00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.996+0000 7fa350c1d900 20 cr:s=0x5631cb8c2c40:op=0x5631cbcb2c00:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.996+0000 7fa350c1d900 15 stack 0x5631cb8c2c40 end 2026-03-20T11:45:58.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.996+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.996+0000 7fa350c1d900 20 run: stack=0x5631cb8c2c40 is done 2026-03-20T11:45:58.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.996+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbcaf9c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.996+0000 7fa350c1d900 20 run: stack=0x5631cbc2ed40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:58.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.996+0000 7fa350c1d900 20 cr:s=0x5631cbd88de0:op=0x5631cbd7bea0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:58.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.996+0000 7fa350c1d900 20 cr:s=0x5631cbd88de0:op=0x5631cbd7bea0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:58.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.996+0000 7fa350c1d900 15 stack 0x5631cbd88de0 end 2026-03-20T11:45:58.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.996+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:58.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.996+0000 7fa350c1d900 20 run: stack=0x5631cbd88de0 is done 2026-03-20T11:45:58.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.996+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbcaf9c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:58.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:58.996+0000 7fa350c1d900 20 run: stack=0x5631cbc2ed40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.003+0000 7fa350c1d900 20 cr:s=0x5631cbd75bc0:op=0x5631cbda67a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.003+0000 7fa350c1d900 20 cr:s=0x5631cbd75bc0:op=0x5631cbda67a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.003+0000 7fa350c1d900 15 stack 0x5631cbd75bc0 end 2026-03-20T11:45:59.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.003+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.003+0000 7fa350c1d900 20 run: stack=0x5631cbd75bc0 is done 2026-03-20T11:45:59.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.003+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbda7d60:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 cr:s=0x5631cbd80510:op=0x5631cbd76ca0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.006 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:59.006+0000 7fd091b71640 -1 req 2266081255560080575 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:59.006 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:59.006+0000 7fd0a3394640 -1 req 17605703335666710571 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:59.006 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:59.008+0000 7fd0c0bcf640 -1 req 7208804320660239288 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 cr:s=0x5631cbd80510:op=0x5631cbd76ca0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 15 stack 0x5631cbd80510 end 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 run: stack=0x5631cbd80510 is done 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 cr:s=0x5631cb8c1740:op=0x5631cbd7e4a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 cr:s=0x5631cb8c1740:op=0x5631cbd7e4a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 15 stack 0x5631cb8c1740 end 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 run: stack=0x5631cb8c1740 is done 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 cr:s=0x5631cbd7b300:op=0x5631cbd72300:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 cr:s=0x5631cbd7b300:op=0x5631cbd72300:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 15 stack 0x5631cbd7b300 end 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 run: stack=0x5631cbd7b300 is done 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbcb6c40:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbda7d60:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbcaf9c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbca5560:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 trim: trimmed bucket instance dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 15 stack 0x5631cbc36db0 end 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is done 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 cr:s=0x5631cbc2ed40:op=0x5631cbca4290:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 trim: trimmed bucket instance dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 15 stack 0x5631cbc2ed40 end 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 run: stack=0x5631cbc2ed40 is done 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbda1dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 4 starting trim on bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbd80c20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77cf0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.008 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 4 starting trim on bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbd80c20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 enqueued request req=0x5631cb8cc640 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 req: 0x5631cb8cc640 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 run: stack=0x5631cbcb3810 is io blocked 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd73890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd73890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 enqueued request req=0x5631cbca89a0 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 req: 0x5631cb8cc640 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 req: 0x5631cbca89a0 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is io blocked 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa3047f8640 20 dequeued request req=0x5631cb8cc640 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa3047f8640 20 RGWWQ: 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa3047f8640 20 req: 0x5631cbca89a0 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa3047f8640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 : miss 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.004+0000 7fa3047f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa303ff7640 20 dequeued request req=0x5631cbca89a0 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa303ff7640 20 RGWWQ: empty 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa303ff7640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 : miss 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa303ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa3047f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa3047f8640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 info.flags=0x17 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa3047f8640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 to cache LRU end 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa3047f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa3047f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa3047f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa3047f8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-25 : miss 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa3047f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa303ff7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa303ff7640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 info.flags=0x17 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa303ff7640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 to cache LRU end 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa303ff7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa303ff7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa303ff7640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa303ff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa303ff7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-35 : miss 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa303ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa303ff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa303ff7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-35 info.flags=0x0 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa303ff7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-35 to cache LRU end 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa303ff7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-35 : miss 2026-03-20T11:45:59.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa303ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa3047f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa3047f8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-25 info.flags=0x0 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa3047f8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-25 to cache LRU end 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa3047f8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-25 : miss 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.005+0000 7fa3047f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa303ff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa303ff7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-35 info.flags=0x0 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa303ff7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-35 to cache LRU end 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa303ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40): adding source pipe: {s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa303ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40): adding dest pipe: {s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa303ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa303ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa303ff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa3047f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa3047f8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-25 info.flags=0x0 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa3047f8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-25 to cache LRU end 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa3047f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30): adding source pipe: {s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa3047f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30): adding dest pipe: {s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd73890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd73890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd73890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd73890:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77cf0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa3047f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa3047f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbda0410:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77cf0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa3047f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cb8ce9e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cb8ce9e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.010 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 6b72344e6585120d25705af6f16d05563d0a9f87f73d9501ca7954344b319127 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:6b72344e6585120d25705af6f16d05563d0a9f87f73d9501ca7954344b319127 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 491838618234108a9fb6ad1e50b4250d5c94db4700d4c19242d8221f159efb18 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=491838618234108a9fb6ad1e50b4250d5c94db4700d4c19242d8221f159efb18 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=12, curl_handle=0x5631cbcad4d0 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 run: stack=0x5631cbcabf70 is io blocked 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbd80c20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbd80c20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbd80c20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbd80c20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbda1dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa2a9ffb640 20 link_request req_data=0x5631cbcadbe0 req_data->id=12, curl_handle=0x5631cbcad4d0 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd80c20:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbda1dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 run: stack=0x5631cbcb3810 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd73890:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd73890:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 351a0ba1dd366709bdc216f02074f37a95556ed0d4eb8bfafc6c5e0fc72c0562 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:351a0ba1dd366709bdc216f02074f37a95556ed0d4eb8bfafc6c5e0fc72c0562 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 950ea83ebfccc9f678f7743979a3e1da4de2981f2b9e8671e2022c4f24ef4aad 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=950ea83ebfccc9f678f7743979a3e1da4de2981f2b9e8671e2022c4f24ef4aad 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=13, curl_handle=0x5631cb8cf450 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is io blocked 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.006+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8cff10 req_data->id=13, curl_handle=0x5631cb8cf450 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cb8ce9e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cb8ce9e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbda0410:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 15 stack 0x5631cbcabf70 end 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 run: stack=0x5631cbcabf70 is done 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd73890:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd73890:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77cf0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 collect(): s=0x5631cbc36db0 stack=0x5631cbcabf70 is complete 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77cf0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 15 stack 0x5631cbc36db0 end 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 stack->operate() returned ret=-22 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is done 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd80c20:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 15 stack 0x5631cbca2ac0 end 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is done 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbda1dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 collect(): s=0x5631cbcb3810 stack=0x5631cbca2ac0 is complete 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbda1dc0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 15 stack 0x5631cbcb3810 end 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 stack->operate() returned ret=-22 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 run: stack=0x5631cbcb3810 is done 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77cf0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 4 starting trim on bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd80c20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbda1dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 4 starting trim on bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd80c20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 enqueued request req=0x5631cbd7e210 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 req: 0x5631cbd7e210 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is io blocked 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbd76ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbd76ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 enqueued request req=0x5631cbca89a0 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 req: 0x5631cbd7e210 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 req: 0x5631cbca89a0 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa350c1d900 20 run: stack=0x5631cbcb3810 is io blocked 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa3037f6640 20 dequeued request req=0x5631cbd7e210 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa3037f6640 20 RGWWQ: 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa3037f6640 20 req: 0x5631cbca89a0 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa3037f6640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 : miss 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa3037f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa302ff5640 20 dequeued request req=0x5631cbca89a0 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa302ff5640 20 RGWWQ: empty 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa302ff5640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 : miss 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa302ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa3037f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa3037f6640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 info.flags=0x17 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa3037f6640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 to cache LRU end 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa3037f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa3037f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa3037f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa3037f6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-26 : miss 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa3037f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa302ff5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa302ff5640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 info.flags=0x17 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa302ff5640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 to cache LRU end 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa302ff5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa302ff5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa302ff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa302ff5640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-51 : miss 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.007+0000 7fa302ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa3037f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa3037f6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-26 info.flags=0x0 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa3037f6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-26 to cache LRU end 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa3037f6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-26 : miss 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa3037f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa302ff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa302ff5640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-51 info.flags=0x0 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa302ff5640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-51 to cache LRU end 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa302ff5640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-51 : miss 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa302ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa3037f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa3037f6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-26 info.flags=0x0 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa3037f6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-26 to cache LRU end 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa3037f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31): adding source pipe: {s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa3037f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31): adding dest pipe: {s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa3037f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa3037f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa3037f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:45:59.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa302ff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd80c20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa302ff5640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-51 info.flags=0x0 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa302ff5640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-51 to cache LRU end 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd80c20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd80c20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd80c20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77cf0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa302ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11): adding source pipe: {s={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cb8bc590:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77cf0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa302ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11): adding dest pipe: {s={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd80060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd80060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa302ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa302ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa302ff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = e7c390165c9809ac53e78e8e756b6e3786d065f005f97221d8aa78b29db0ec9a 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:e7c390165c9809ac53e78e8e756b6e3786d065f005f97221d8aa78b29db0ec9a 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 1fff20a72cdfb800597b49f7d1d36dfd21b6b62363d451420780e02238475904 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=1fff20a72cdfb800597b49f7d1d36dfd21b6b62363d451420780e02238475904 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=14, curl_handle=0x5631cb8cf450 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is io blocked 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbd76ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbd76ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbd76ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbd76ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbda1dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cb8ce9e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8cff10 req_data->id=14, curl_handle=0x5631cb8cf450 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbda1dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 run: stack=0x5631cbcb3810 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbcb6c40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbcb6c40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 761e18dccbba1142e45c5075460edade18e03cabee0607a140022be89b1525f0 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:761e18dccbba1142e45c5075460edade18e03cabee0607a140022be89b1525f0 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 22e42d76d2d6cf7c708c20bc9098efb4953d17eb06dad4691c9c2fa3ce69c25a 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=22e42d76d2d6cf7c708c20bc9098efb4953d17eb06dad4691c9c2fa3ce69c25a 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=15, curl_handle=0x5631cbcad4d0 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 run: stack=0x5631cbcabf70 is io blocked 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa2a9ffb640 20 link_request req_data=0x5631cbcadbe0 req_data->id=15, curl_handle=0x5631cbcad4d0 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd79fc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd79fc0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 15 stack 0x5631cbcb6b00 end 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is done 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbda7d60:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 trim: trimmed bucket instance dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 15 stack 0x5631cbd5d230 end 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is done 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 4 starting trim on bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbd80c20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbd80c20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 enqueued request req=0x5631cbca3980 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 req: 0x5631cbca3980 2026-03-20T11:45:59.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is io blocked 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cb8bfd90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cb8bfd90:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 15 stack 0x5631cbcb7f60 end 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 run: stack=0x5631cbcb7f60 is done 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd74f70:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbca6830:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 trim: trimmed bucket instance dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 15 stack 0x5631cbcb4470 end 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa3027f4640 20 dequeued request req=0x5631cbca3980 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa3027f4640 20 RGWWQ: empty 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 run: stack=0x5631cbcb4470 is done 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 4 starting trim on bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd79fc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd79fc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 enqueued request req=0x5631cbd776f0 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 req: 0x5631cbd776f0 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa350c1d900 20 run: stack=0x5631cbcb4470 is io blocked 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa3027f4640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 : miss 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa3027f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa301ff3640 20 dequeued request req=0x5631cbd776f0 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa301ff3640 20 RGWWQ: empty 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa301ff3640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 : miss 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.008+0000 7fa301ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.009+0000 7fa3027f4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.009+0000 7fa3027f4640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 info.flags=0x17 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.009+0000 7fa3027f4640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 to cache LRU end 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.009+0000 7fa3027f4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.009+0000 7fa3027f4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.009+0000 7fa3027f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.009+0000 7fa3027f4640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-55 : miss 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.009+0000 7fa3027f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa301ff3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa301ff3640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 info.flags=0x17 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa301ff3640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 to cache LRU end 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa301ff3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa301ff3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa301ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa301ff3640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-28 : miss 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa301ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa3027f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa3027f4640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-55 info.flags=0x0 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa3027f4640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-55 to cache LRU end 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa3027f4640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-55 : miss 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa3027f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa301ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa301ff3640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-28 info.flags=0x0 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa301ff3640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-28 to cache LRU end 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa301ff3640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-28 : miss 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa301ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa3027f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa3027f4640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-55 info.flags=0x0 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa3027f4640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-55 to cache LRU end 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa3027f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding source pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa3027f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding dest pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa3027f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa3027f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa3027f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbd80c20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbd80c20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbd80c20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbd80c20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbd80c20:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cb8bfd90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cb8bfd90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 9e84eee45b099626a1fc6b3623bb08f3be3f8d3f849e1042559e8169d75cdfa6 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:9e84eee45b099626a1fc6b3623bb08f3be3f8d3f849e1042559e8169d75cdfa6 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 7225c9d49a1f0dbcb14513583c433ed42542a6eae9ed305e18a4b8b89f20d8f6 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=7225c9d49a1f0dbcb14513583c433ed42542a6eae9ed305e18a4b8b89f20d8f6 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=16, curl_handle=0x5631cb8c1990 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa350c1d900 20 run: stack=0x5631cbcb7f60 is io blocked 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.010+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8c20a0 req_data->id=16, curl_handle=0x5631cb8c1990 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa301ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa301ff3640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-28 info.flags=0x0 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa301ff3640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-28 to cache LRU end 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa301ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding source pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa301ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding dest pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa301ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa301ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa301ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd79fc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd79fc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd79fc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd79fc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd79fc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 20 run: stack=0x5631cbcb4470 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd76ca0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd76ca0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = ddeeaa7fabd4b7e060d9de53ae978ecd4be19bd1c1c0c7b5c10203a5569c04f2 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:ddeeaa7fabd4b7e060d9de53ae978ecd4be19bd1c1c0c7b5c10203a5569c04f2 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 38a359da496203edce9d4503e9d2658f41316d546a05e348e5e576e4d2dd3754 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=38a359da496203edce9d4503e9d2658f41316d546a05e348e5e576e4d2dd3754 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=17, curl_handle=0x5631cb8bd9e0 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is io blocked 2026-03-20T11:45:59.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.011+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8be0f0 req_data->id=17, curl_handle=0x5631cb8bd9e0 2026-03-20T11:45:59.018 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:59.010+0000 7fd081b51640 -1 req 6849943169334040626 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:59.018 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:59.013+0000 7fd0b6bbb640 -1 req 6655354047254929036 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:59.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd76ca0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd76ca0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:59.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd79fc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 15 stack 0x5631cbcb6b00 end 2026-03-20T11:45:59.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is done 2026-03-20T11:45:59.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 collect(): s=0x5631cbcb4470 stack=0x5631cbcb6b00 is complete 2026-03-20T11:45:59.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:59.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:59.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:59.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 15 stack 0x5631cbcb4470 end 2026-03-20T11:45:59.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 stack->operate() returned ret=-22 2026-03-20T11:45:59.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 run: stack=0x5631cbcb4470 is done 2026-03-20T11:45:59.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:59.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 4 starting trim on bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:45:59.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cb8cd520:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cb8cd520:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 enqueued request req=0x5631cbd776f0 2026-03-20T11:45:59.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 req: 0x5631cbd776f0 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 run: stack=0x5631cbcb4470 is io blocked 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa3017f2640 20 dequeued request req=0x5631cbd776f0 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa3017f2640 20 RGWWQ: empty 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa3017f2640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 : miss 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa3017f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cb8bfd90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cb8bfd90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cb8bfd90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cb8bfd90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbd80c20:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 15 stack 0x5631cbcb7f60 end 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 run: stack=0x5631cbcb7f60 is done 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 collect(): s=0x5631cbd5d230 stack=0x5631cbcb7f60 is complete 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 10 trimming bilogs for bucket=:dipepu-55[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15]) markers=[,,,,,00000000006.98.6,,,,,], shards=11 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbda0fc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 10 trimming bilog shard 5 of :dipepu-55[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15]) at marker 00000000006.98.6 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cb8bfd90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cb8bfd90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15.5 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa350c1d900 20 run: stack=0x5631cbcb7f60 is io blocked 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa3017f2640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa3017f2640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 info.flags=0x17 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa3017f2640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 to cache LRU end 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa3017f2640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa3017f2640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa3017f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa3017f2640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-31 : miss 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa3017f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa3017f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa3017f2640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-31 info.flags=0x0 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa3017f2640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-31 to cache LRU end 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa3017f2640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-31 : miss 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.012+0000 7fa3017f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa3017f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa3017f2640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-31 info.flags=0x0 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa3017f2640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-31 to cache LRU end 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa3017f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding source pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa3017f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding dest pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa3017f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa3017f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa3017f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cb8cd520:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cb8cd520:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cb8cd520:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cb8cd520:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cb8cd520:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 run: stack=0x5631cbcb4470 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd76ca0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd76ca0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.020 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 86fe8f985049522a6bcd35391168becde9cfe787d51af432a1ac43782b71606a 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:86fe8f985049522a6bcd35391168becde9cfe787d51af432a1ac43782b71606a 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 5bee32c3420ee3b3deb502349bb4678ebc8a6f96581c61b3ff10b4dfa6af276b 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5bee32c3420ee3b3deb502349bb4678ebc8a6f96581c61b3ff10b4dfa6af276b 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=18, curl_handle=0x5631cb8c1990 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8c20a0 req_data->id=18, curl_handle=0x5631cb8c1990 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is io blocked 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cb8bfd90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cb8bfd90:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 15 stack 0x5631cbcb7f60 end 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 run: stack=0x5631cbcb7f60 is done 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbda0fc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 trim: trimmed bucket instance dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 15 stack 0x5631cbd5d230 end 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is done 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 4 starting trim on bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cb8bfd90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cb8bfd90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 enqueued request req=0x5631cbca3980 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 req: 0x5631cbca3980 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is io blocked 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa300ff1640 20 dequeued request req=0x5631cbca3980 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa300ff1640 20 RGWWQ: empty 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa300ff1640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 : miss 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.013+0000 7fa300ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa300ff1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa300ff1640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 info.flags=0x17 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa300ff1640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 to cache LRU end 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa300ff1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa300ff1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa300ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa300ff1640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-39 : miss 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa300ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa300ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa300ff1640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-39 info.flags=0x0 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa300ff1640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-39 to cache LRU end 2026-03-20T11:45:59.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa300ff1640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-39 : miss 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa300ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa300ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa300ff1640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-39 info.flags=0x0 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa300ff1640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-39 to cache LRU end 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa300ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding source pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa300ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding dest pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa300ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa300ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa300ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cb8bfd90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cb8bfd90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cb8bfd90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cb8bfd90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbda9b80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cb8bfd90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cb8bfd90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 707caba8a50b725db2bf80258348c7faf50a9e8d2ec313ca8d96e26699d41ece 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:707caba8a50b725db2bf80258348c7faf50a9e8d2ec313ca8d96e26699d41ece 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = d9556fa683d5539f97b3be8c8fb5b4ff57242930ef5d1d0f18c7af6c984b4771 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=d9556fa683d5539f97b3be8c8fb5b4ff57242930ef5d1d0f18c7af6c984b4771 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=19, curl_handle=0x5631cb8bd9e0 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8be0f0 req_data->id=19, curl_handle=0x5631cb8bd9e0 2026-03-20T11:45:59.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.014+0000 7fa350c1d900 20 run: stack=0x5631cbcb7f60 is io blocked 2026-03-20T11:45:59.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd80060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd80060:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:59.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cb8bc590:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa350c1d900 15 stack 0x5631cbca2ac0 end 2026-03-20T11:45:59.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is done 2026-03-20T11:45:59.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77cf0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa350c1d900 20 collect(): s=0x5631cbc36db0 stack=0x5631cbca2ac0 is complete 2026-03-20T11:45:59.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa350c1d900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:59.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa350c1d900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:59.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77cf0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:59.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa350c1d900 15 stack 0x5631cbc36db0 end 2026-03-20T11:45:59.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa350c1d900 20 stack->operate() returned ret=-22 2026-03-20T11:45:59.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is done 2026-03-20T11:45:59.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa350c1d900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77cf0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa350c1d900 4 starting trim on bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd84f50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd84f50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa350c1d900 20 enqueued request req=0x5631cbd7e210 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa350c1d900 20 req: 0x5631cbd7e210 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is io blocked 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa3007f0640 20 dequeued request req=0x5631cbd7e210 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa3007f0640 20 RGWWQ: empty 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa3007f0640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 : miss 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa3007f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa3007f0640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa3007f0640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 info.flags=0x17 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa3007f0640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 to cache LRU end 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa3007f0640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa3007f0640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa3007f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa3007f0640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-8 : miss 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.049+0000 7fa3007f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa3007f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa3007f0640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-8 info.flags=0x0 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa3007f0640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-8 to cache LRU end 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa3007f0640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-8 : miss 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa3007f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa3007f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa3007f0640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-8 info.flags=0x0 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa3007f0640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-8 to cache LRU end 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa3007f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding source pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa3007f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding dest pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa3007f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa3007f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa3007f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd84f50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd84f50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd84f50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd84f50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77cf0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cb8bc590:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77cf0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd80060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd80060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 140c6ee04c8bcb962d156091fa557bb69dd1ee2441d3927069ad46298d9f1d90 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:140c6ee04c8bcb962d156091fa557bb69dd1ee2441d3927069ad46298d9f1d90 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = f3d52b79451b1b9a948db3284f6dd9afe730aca8f3444071b71f68d35ee6ce33 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=f3d52b79451b1b9a948db3284f6dd9afe730aca8f3444071b71f68d35ee6ce33 2026-03-20T11:45:59.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=20, curl_handle=0x5631cb8cf450 2026-03-20T11:45:59.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8cff10 req_data->id=20, curl_handle=0x5631cb8cf450 2026-03-20T11:45:59.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.050+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is io blocked 2026-03-20T11:45:59.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbcb6c40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbcb6c40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbcb6c40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbcb6c40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cb8ce9e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 15 stack 0x5631cbcabf70 end 2026-03-20T11:45:59.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 run: stack=0x5631cbcabf70 is done 2026-03-20T11:45:59.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbda1dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 collect(): s=0x5631cbcb3810 stack=0x5631cbcabf70 is complete 2026-03-20T11:45:59.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 10 trimming bilogs for bucket=:dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) markers=[00000000004.41.6,00000000002.16.1,00000000004.28.6,00000000002.34.1,00000000004.45.6,00000000002.40.1,00000000002.39.1,00000000004.52.6,00000000002.73.1,00000000002.41.1,00000000002.74.1], shards=11 2026-03-20T11:45:59.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbda0fc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 10 trimming bilog shard 0 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000004.41.6 2026-03-20T11:45:59.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 10 trimming bilog shard 1 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.16.1 2026-03-20T11:45:59.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 10 trimming bilog shard 2 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000004.28.6 2026-03-20T11:45:59.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 10 trimming bilog shard 3 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.34.1 2026-03-20T11:45:59.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 10 trimming bilog shard 4 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000004.45.6 2026-03-20T11:45:59.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 10 trimming bilog shard 5 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.40.1 2026-03-20T11:45:59.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 10 trimming bilog shard 6 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.39.1 2026-03-20T11:45:59.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 10 trimming bilog shard 7 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000004.52.6 2026-03-20T11:45:59.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 10 trimming bilog shard 8 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.73.1 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 10 trimming bilog shard 9 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.41.1 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 10 trimming bilog shard 10 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.74.1 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 run: stack=0x5631cbcb3810 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbda67a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbda7040:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cbda4c60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbd85820:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda02b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbd9f300:op=0x5631cbda3150:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbda3ea0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cbd75bc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbca6e20:op=0x5631cbcb5a80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbd7a470:op=0x5631cbca5ba0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbd730f0:op=0x5631cbca63a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbda67a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.0 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 run: stack=0x5631cbcabf70 is io blocked 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbda7040:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.1 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 run: stack=0x5631cbca8810 is io blocked 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cbda4c60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.2 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 run: stack=0x5631cbca9120 is io blocked 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbd85820:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.3 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 run: stack=0x5631cbd79340 is io blocked 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda02b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.4 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is io blocked 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbd9f300:op=0x5631cbda3150:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.5 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 run: stack=0x5631cbd9f300 is io blocked 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbda3ea0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.6 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 run: stack=0x5631cbd72b00 is io blocked 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cbd75bc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.7 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 run: stack=0x5631cbd718c0 is io blocked 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbca6e20:op=0x5631cbcb5a80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.8 2026-03-20T11:45:59.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 run: stack=0x5631cbca6e20 is io blocked 2026-03-20T11:45:59.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbd7a470:op=0x5631cbca5ba0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.9 2026-03-20T11:45:59.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 run: stack=0x5631cbd7a470 is io blocked 2026-03-20T11:45:59.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 cr:s=0x5631cbd730f0:op=0x5631cbca63a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.10 2026-03-20T11:45:59.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.051+0000 7fa350c1d900 20 run: stack=0x5631cbd730f0 is io blocked 2026-03-20T11:45:59.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbda67a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbda67a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 15 stack 0x5631cbcabf70 end 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 20 run: stack=0x5631cbcabf70 is done 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbda0fc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 20 run: stack=0x5631cbcb3810 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbda3ea0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbda3ea0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 15 stack 0x5631cbd72b00 end 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 20 run: stack=0x5631cbd72b00 is done 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbda0fc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 20 run: stack=0x5631cbcb3810 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cbd75bc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cbd75bc0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 15 stack 0x5631cbd718c0 end 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 20 run: stack=0x5631cbd718c0 is done 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cbda4c60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cbda4c60:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 15 stack 0x5631cbca9120 end 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 20 run: stack=0x5631cbca9120 is done 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbd85820:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbd85820:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 15 stack 0x5631cbd79340 end 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 20 run: stack=0x5631cbd79340 is done 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbda0fc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbda0fc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbda0fc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.053+0000 7fa350c1d900 20 run: stack=0x5631cbcb3810 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbda7040:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbda7040:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 15 stack 0x5631cbca8810 end 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 run: stack=0x5631cbca8810 is done 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbda0fc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 run: stack=0x5631cbcb3810 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd76ca0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd76ca0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cb8cd520:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 15 stack 0x5631cbcb6b00 end 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is done 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 collect(): s=0x5631cbcb4470 stack=0x5631cbcb6b00 is complete 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 cr:s=0x5631cbcb4470:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 15 stack 0x5631cbcb4470 end 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 stack->operate() returned ret=-22 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 run: stack=0x5631cbcb4470 is done 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 4 starting trim on bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbda67a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbda67a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 enqueued request req=0x5631cbc36b80 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 req: 0x5631cbc36b80 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is io blocked 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa2fffef640 20 dequeued request req=0x5631cbc36b80 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa2fffef640 20 RGWWQ: empty 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa2fffef640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 : miss 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa2fffef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda02b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda02b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 15 stack 0x5631cbd84c00 end 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is done 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbda0fc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 run: stack=0x5631cbcb3810 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 cr:s=0x5631cbd9f300:op=0x5631cbda3150:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 cr:s=0x5631cbd9f300:op=0x5631cbda3150:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 15 stack 0x5631cbd9f300 end 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 run: stack=0x5631cbd9f300 is done 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbda0fc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa350c1d900 20 run: stack=0x5631cbcb3810 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa2fffef640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa2fffef640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 info.flags=0x17 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa2fffef640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 to cache LRU end 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa2fffef640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.054+0000 7fa2fffef640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2fffef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2fffef640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-16 : miss 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2fffef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 cr:s=0x5631cbca6e20:op=0x5631cbcb5a80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 cr:s=0x5631cbca6e20:op=0x5631cbcb5a80:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 15 stack 0x5631cbca6e20 end 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 run: stack=0x5631cbca6e20 is done 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbda0fc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 run: stack=0x5631cbcb3810 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 cr:s=0x5631cbd7a470:op=0x5631cbca5ba0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 cr:s=0x5631cbd7a470:op=0x5631cbca5ba0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 15 stack 0x5631cbd7a470 end 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 run: stack=0x5631cbd7a470 is done 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbda0fc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 run: stack=0x5631cbcb3810 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 cr:s=0x5631cbd730f0:op=0x5631cbca63a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 cr:s=0x5631cbd730f0:op=0x5631cbca63a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 15 stack 0x5631cbd730f0 end 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 run: stack=0x5631cbd730f0 is done 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbda0fc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 cr:s=0x5631cbcb3810:op=0x5631cbda1dc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 trim: trimmed bucket instance dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 15 stack 0x5631cbcb3810 end 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 run: stack=0x5631cbcb3810 is done 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda7a60:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 4 starting trim on bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cb8cdc90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cb8cdc90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 enqueued request req=0x5631cbd71a80 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 req: 0x5631cbd71a80 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is io blocked 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2ff7ee640 20 dequeued request req=0x5631cbd71a80 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2ff7ee640 20 RGWWQ: empty 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2ff7ee640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 : miss 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2ff7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2ff7ee640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=314 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2ff7ee640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 info.flags=0x17 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2ff7ee640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 to cache LRU end 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2ff7ee640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2ff7ee640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2ff7ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2ff7ee640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-43 : miss 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2ff7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2fffef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2fffef640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-16 info.flags=0x0 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2fffef640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-16 to cache LRU end 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2fffef640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-16 : miss 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2fffef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2ff7ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2ff7ee640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-43 info.flags=0x0 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2ff7ee640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-43 to cache LRU end 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2ff7ee640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-43 : miss 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2ff7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2fffef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2fffef640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-16 info.flags=0x0 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2fffef640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-16 to cache LRU end 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2fffef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding source pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2fffef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding dest pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2fffef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2fffef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2fffef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbda67a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbda67a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbda67a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbda67a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2ff7ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2ff7ee640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-43 info.flags=0x0 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2ff7ee640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-43 to cache LRU end 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd76ca0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2ff7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding source pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2ff7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding dest pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cb8cd520:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cb8cd520:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.055+0000 7fa2ff7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa2ff7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa2ff7ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.055 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 2eb50db2b411055d89f93805c73ff1b3d461d6d0cf8e84e9b1a8bc11d07ba2d8 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2eb50db2b411055d89f93805c73ff1b3d461d6d0cf8e84e9b1a8bc11d07ba2d8 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 2c4c600768d21d914dd2f76bbac4d3695149c4ff301a09406df7f6e359c3be22 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2c4c600768d21d914dd2f76bbac4d3695149c4ff301a09406df7f6e359c3be22 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=21, curl_handle=0x5631cb8c1990 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8c20a0 req_data->id=21, curl_handle=0x5631cb8c1990 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 20 run: stack=0x5631cbca8810 is io blocked 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cb8cdc90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cb8cdc90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cb8cdc90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cb8cdc90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda7a60:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cbcb6c40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda7a60:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cb8cdc90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cb8cdc90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 8ad5053a055957aaaeb3d262302e3b6eca8964319bedd80a5a822dd0a88bd8b0 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:8ad5053a055957aaaeb3d262302e3b6eca8964319bedd80a5a822dd0a88bd8b0 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 92cca53e6ed627fc350b0be2e78c9ac855b56f8c6d03b5625b53823c8d9af344 2026-03-20T11:45:59.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=92cca53e6ed627fc350b0be2e78c9ac855b56f8c6d03b5625b53823c8d9af344 2026-03-20T11:45:59.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=22, curl_handle=0x5631cbcad4d0 2026-03-20T11:45:59.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa2a9ffb640 20 link_request req_data=0x5631cbcadbe0 req_data->id=22, curl_handle=0x5631cbcad4d0 2026-03-20T11:45:59.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.056+0000 7fa350c1d900 20 run: stack=0x5631cbd718c0 is io blocked 2026-03-20T11:45:59.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.057+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cb8bfd90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.057+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cb8bfd90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.057+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cb8bfd90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.057+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cb8bfd90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.057+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbda9b80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.057+0000 7fa350c1d900 15 stack 0x5631cbcb7f60 end 2026-03-20T11:45:59.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.057+0000 7fa350c1d900 20 run: stack=0x5631cbcb7f60 is done 2026-03-20T11:45:59.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.057+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.057+0000 7fa350c1d900 20 collect(): s=0x5631cbd5d230 stack=0x5631cbcb7f60 is complete 2026-03-20T11:45:59.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.057+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.057+0000 7fa350c1d900 10 trimming bilogs for bucket=:dipepu-39[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44]) markers=[,,,,,,,,00000000001.10.6,,], shards=11 2026-03-20T11:45:59.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.057+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbdb3a80:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.057+0000 7fa350c1d900 10 trimming bilog shard 8 of :dipepu-39[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44]) at marker 00000000001.10.6 2026-03-20T11:45:59.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.057+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.057+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbda99c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.057+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbda99c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.057+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44.8 2026-03-20T11:45:59.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.057+0000 7fa350c1d900 20 run: stack=0x5631cbcb7f60 is io blocked 2026-03-20T11:45:59.057 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:59.055+0000 7fd091370640 -1 req 12499271438345817625 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbda99c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbda99c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa350c1d900 15 stack 0x5631cbcb7f60 end 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa350c1d900 20 run: stack=0x5631cbcb7f60 is done 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbdb3a80:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa350c1d900 20 trim: trimmed bucket instance dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa350c1d900 15 stack 0x5631cbd5d230 end 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is done 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa350c1d900 4 starting trim on bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbda28e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbda28e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa350c1d900 20 enqueued request req=0x5631cbca3980 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa350c1d900 20 req: 0x5631cbca3980 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is io blocked 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa2fefed640 20 dequeued request req=0x5631cbca3980 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa2fefed640 20 RGWWQ: empty 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa2fefed640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 : miss 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa2fefed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa2fefed640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa2fefed640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 info.flags=0x17 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa2fefed640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 to cache LRU end 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa2fefed640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa2fefed640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa2fefed640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa2fefed640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-10 : miss 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.058+0000 7fa2fefed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa2fefed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa2fefed640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-10 info.flags=0x0 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa2fefed640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-10 to cache LRU end 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa2fefed640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-10 : miss 2026-03-20T11:45:59.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa2fefed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa2fefed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa2fefed640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-10 info.flags=0x0 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa2fefed640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-10 to cache LRU end 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa2fefed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding source pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa2fefed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding dest pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa2fefed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa2fefed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa2fefed640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbda28e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbda28e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbda28e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbda28e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbcb5a60:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbda3e40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbda3e40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 3cf2a746d1d16570be855ac0dfaa1ede74f5c90c87e4441aeb0c131fc1e67f1d 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:3cf2a746d1d16570be855ac0dfaa1ede74f5c90c87e4441aeb0c131fc1e67f1d 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 49aaee2db4d716929dbdd8ea4fcd50c000b96370396846b6757e02b97525d9e1 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=49aaee2db4d716929dbdd8ea4fcd50c000b96370396846b6757e02b97525d9e1 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=23, curl_handle=0x5631cb8bd9e0 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8be0f0 req_data->id=23, curl_handle=0x5631cb8bd9e0 2026-03-20T11:45:59.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.059+0000 7fa350c1d900 20 run: stack=0x5631cbcb7f60 is io blocked 2026-03-20T11:45:59.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd80060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd80060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd80060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd80060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cb8bc590:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 15 stack 0x5631cbca2ac0 end 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is done 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77cf0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 collect(): s=0x5631cbc36db0 stack=0x5631cbca2ac0 is complete 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 10 trimming bilogs for bucket=:dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) markers=[,00000000002.12.6,,00000000001.5.6,,00000000001.4.6,00000000002.12.6,00000000001.8.6,00000000001.14.6,00000000001.9.6,], shards=11 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdb3a80:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 10 trimming bilog shard 1 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000002.12.6 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 10 trimming bilog shard 3 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.5.6 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 10 trimming bilog shard 5 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.4.6 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 10 trimming bilog shard 6 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000002.12.6 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 10 trimming bilog shard 7 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.8.6 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 10 trimming bilog shard 8 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.14.6 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 10 trimming bilog shard 9 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.9.6 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd84f50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbca6d80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cbda3150:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbd7cfe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbd7fcc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 cr:s=0x5631cbcb3260:op=0x5631cbca55c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 cr:s=0x5631cbd72e40:op=0x5631cbda2300:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd84f50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.1 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is io blocked 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbca6d80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.3 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 run: stack=0x5631cbd79340 is io blocked 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cbda3150:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.5 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 run: stack=0x5631cbca9120 is io blocked 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbd7cfe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.6 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 run: stack=0x5631cbd72b00 is io blocked 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbd7fcc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.7 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 run: stack=0x5631cbcabf70 is io blocked 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 cr:s=0x5631cbcb3260:op=0x5631cbca55c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.8 2026-03-20T11:45:59.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 run: stack=0x5631cbcb3260 is io blocked 2026-03-20T11:45:59.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 cr:s=0x5631cbd72e40:op=0x5631cbda2300:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.9 2026-03-20T11:45:59.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.094+0000 7fa350c1d900 20 run: stack=0x5631cbd72e40 is io blocked 2026-03-20T11:45:59.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.095+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cbda3150:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.095+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cbda3150:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.095+0000 7fa350c1d900 15 stack 0x5631cbca9120 end 2026-03-20T11:45:59.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.095+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.095+0000 7fa350c1d900 20 run: stack=0x5631cbca9120 is done 2026-03-20T11:45:59.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.095+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdb3a80:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.095+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbca6d80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbca6d80:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 15 stack 0x5631cbd79340 end 2026-03-20T11:45:59.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 run: stack=0x5631cbd79340 is done 2026-03-20T11:45:59.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdb3a80:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 cr:s=0x5631cbcb3260:op=0x5631cbca55c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 cr:s=0x5631cbcb3260:op=0x5631cbca55c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 15 stack 0x5631cbcb3260 end 2026-03-20T11:45:59.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 run: stack=0x5631cbcb3260 is done 2026-03-20T11:45:59.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdb3a80:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbd7cfe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbd7cfe0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 15 stack 0x5631cbd72b00 end 2026-03-20T11:45:59.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 run: stack=0x5631cbd72b00 is done 2026-03-20T11:45:59.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdb3a80:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 cr:s=0x5631cbd72e40:op=0x5631cbda2300:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 cr:s=0x5631cbd72e40:op=0x5631cbda2300:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 15 stack 0x5631cbd72e40 end 2026-03-20T11:45:59.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 run: stack=0x5631cbd72e40 is done 2026-03-20T11:45:59.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdb3a80:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd84f50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd84f50:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 15 stack 0x5631cbca2ac0 end 2026-03-20T11:45:59.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is done 2026-03-20T11:45:59.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdb3a80:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbd7fcc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbd7fcc0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 15 stack 0x5631cbcabf70 end 2026-03-20T11:45:59.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 run: stack=0x5631cbcabf70 is done 2026-03-20T11:45:59.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdb3a80:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77cf0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 trim: trimmed bucket instance dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 15 stack 0x5631cbc36db0 end 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is done 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77cf0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 4 starting trim on bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd868f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd868f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 enqueued request req=0x5631cb8cc640 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 req: 0x5631cb8cc640 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is io blocked 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa2fe7ec640 20 dequeued request req=0x5631cb8cc640 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa2fe7ec640 20 RGWWQ: empty 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa2fe7ec640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 : miss 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa2fe7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa2fe7ec640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa2fe7ec640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 info.flags=0x17 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa2fe7ec640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 to cache LRU end 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa2fe7ec640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa2fe7ec640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa2fe7ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa2fe7ec640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-3 : miss 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.096+0000 7fa2fe7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cb8cdc90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cb8cdc90:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cbcb6c40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 15 stack 0x5631cbd718c0 end 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 run: stack=0x5631cbd718c0 is done 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda7a60:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 collect(): s=0x5631cbd84c00 stack=0x5631cbd718c0 is complete 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fe7ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda7a60:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:59.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fe7ec640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-3 info.flags=0x0 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 15 stack 0x5631cbd84c00 end 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fe7ec640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-3 to cache LRU end 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 stack->operate() returned ret=-22 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is done 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fe7ec640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-3 : miss 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fe7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda7040:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 4 starting trim on bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda3150:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda3150:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 enqueued request req=0x5631cbd71a80 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 req: 0x5631cbd71a80 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is io blocked 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fdfeb640 20 dequeued request req=0x5631cbd71a80 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fdfeb640 20 RGWWQ: empty 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fdfeb640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 : miss 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fdfeb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fe7ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fe7ec640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-3 info.flags=0x0 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fe7ec640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-3 to cache LRU end 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fe7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding source pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fe7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding dest pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fe7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fe7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fe7ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd868f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd868f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd868f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd868f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77cf0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cb8bc590:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.097 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77cf0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.097 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:59.097+0000 7fd0b93c0640 -1 req 6066074214859425674 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:59.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cbca4740:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cbca4740:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.103 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.103 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.103 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.103 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.103 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.103 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.103 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.103 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.103 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.103 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.103 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.103 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.103 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 29e5787122f4b75db9453de6be157aa221c7b5a4faed51ccb6dc3bac0d9ea8d5 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:29e5787122f4b75db9453de6be157aa221c7b5a4faed51ccb6dc3bac0d9ea8d5 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = c9c9b9578c7bddfd2dcf998889b3de1fcd3cade229e8259bde2e396ab2f75ec1 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c9c9b9578c7bddfd2dcf998889b3de1fcd3cade229e8259bde2e396ab2f75ec1 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=24, curl_handle=0x5631cbcad4d0 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2a9ffb640 20 link_request req_data=0x5631cbcadbe0 req_data->id=24, curl_handle=0x5631cbcad4d0 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fdfeb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fdfeb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 info.flags=0x17 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fdfeb640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 to cache LRU end 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fdfeb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fdfeb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fdfeb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fdfeb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-9 : miss 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fdfeb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa350c1d900 20 run: stack=0x5631cbd718c0 is io blocked 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fdfeb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fdfeb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-9 info.flags=0x0 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fdfeb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-9 to cache LRU end 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fdfeb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-9 : miss 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.097+0000 7fa2fdfeb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa2fdfeb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa2fdfeb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-9 info.flags=0x0 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa2fdfeb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-9 to cache LRU end 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa2fdfeb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding source pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa2fdfeb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding dest pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa2fdfeb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa2fdfeb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa2fdfeb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda3150:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda3150:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cb8cd520:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda3150:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cb8cd520:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda3150:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cb8cd520:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda7040:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cb8cd520:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbda3150:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda7040:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd76ca0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 15 stack 0x5631cbca8810 end 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 run: stack=0x5631cbca8810 is done 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbca5c20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 collect(): s=0x5631cbcb6b00 stack=0x5631cbca8810 is complete 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 10 trimming bilogs for bucket=:dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) markers=[,,,,,00000000003.17.10,,,00000000031.93.9,,], shards=11 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbca5c20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 522798cc93fab61c4993fd2c2aff8a88b171d0254c7e8b975a926d154e8b1928 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:522798cc93fab61c4993fd2c2aff8a88b171d0254c7e8b975a926d154e8b1928 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = b6b1dc18fbb53b01952fe9eecc7800fb985b5e7c3e01f0e1946c3031662a7d87 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b6b1dc18fbb53b01952fe9eecc7800fb985b5e7c3e01f0e1946c3031662a7d87 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=25, curl_handle=0x5631cb8c1990 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8c20a0 req_data->id=25, curl_handle=0x5631cb8c1990 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 run: stack=0x5631cbcabf70 is io blocked 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cb8cd520:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 10 trimming bilog shard 5 of :dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) at marker 00000000003.17.10 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 10 trimming bilog shard 8 of :dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) at marker 00000000031.93.9 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd76ca0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbda67a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd76ca0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21.5 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 run: stack=0x5631cbca8810 is io blocked 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbda67a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21.8 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.098+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is io blocked 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbda67a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbda67a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa350c1d900 15 stack 0x5631cbca2ac0 end 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is done 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cb8cd520:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd76ca0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd76ca0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa350c1d900 15 stack 0x5631cbca8810 end 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa350c1d900 20 run: stack=0x5631cbca8810 is done 2026-03-20T11:45:59.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cb8cd520:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa350c1d900 20 trim: trimmed bucket instance dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa350c1d900 15 stack 0x5631cbcb6b00 end 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is done 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa350c1d900 4 starting trim on bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbca54a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbca54a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa350c1d900 20 enqueued request req=0x5631cbd71a80 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa350c1d900 20 req: 0x5631cbd71a80 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is io blocked 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa2fd7ea640 20 dequeued request req=0x5631cbd71a80 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa2fd7ea640 20 RGWWQ: empty 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa2fd7ea640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 : miss 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa2fd7ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa2fd7ea640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa2fd7ea640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 info.flags=0x17 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa2fd7ea640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 to cache LRU end 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa2fd7ea640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa2fd7ea640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa2fd7ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa2fd7ea640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-38 : miss 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.099+0000 7fa2fd7ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa2fd7ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa2fd7ea640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-38 info.flags=0x0 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa2fd7ea640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-38 to cache LRU end 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa2fd7ea640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-38 : miss 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa2fd7ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa2fd7ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa2fd7ea640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-38 info.flags=0x0 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa2fd7ea640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-38 to cache LRU end 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa2fd7ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding source pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa2fd7ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding dest pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa2fd7ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa2fd7ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa2fd7ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbca54a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbca54a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbca54a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbca54a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cb8cd520:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbca54a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbca54a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.106 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 619899ad7268aff7e2ecf3836da403c52c1ede2d97ac6b7d491568c4b6b35653 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:619899ad7268aff7e2ecf3836da403c52c1ede2d97ac6b7d491568c4b6b35653 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = c399b4074613583984e1ebc11838aed0f2a57f26c894cdffafde9ed4e982398a 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c399b4074613583984e1ebc11838aed0f2a57f26c894cdffafde9ed4e982398a 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=26, curl_handle=0x5631cb8cf450 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8cff10 req_data->id=26, curl_handle=0x5631cb8cf450 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.100+0000 7fa350c1d900 20 run: stack=0x5631cbca8810 is io blocked 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.101+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbda3e40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.101+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbda3e40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.101+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbda3e40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.101+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbda3e40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.101+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbcb5a60:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.101+0000 7fa350c1d900 15 stack 0x5631cbcb7f60 end 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.101+0000 7fa350c1d900 20 run: stack=0x5631cbcb7f60 is done 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.101+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.101+0000 7fa350c1d900 20 collect(): s=0x5631cbd5d230 stack=0x5631cbcb7f60 is complete 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.101+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.101+0000 7fa350c1d900 10 trimming bilogs for bucket=:dipepu-10[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15]) markers=[,00000000002.9.6,,,,,,,,,], shards=11 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.101+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbd74080:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.101+0000 7fa350c1d900 10 trimming bilog shard 1 of :dipepu-10[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15]) at marker 00000000002.9.6 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.101+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.101+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbda99c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.101+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbda99c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.101+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15.1 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.101+0000 7fa350c1d900 20 run: stack=0x5631cbcb7f60 is io blocked 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbda99c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbda99c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 15 stack 0x5631cbcb7f60 end 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 run: stack=0x5631cbcb7f60 is done 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbd74080:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 trim: trimmed bucket instance dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 15 stack 0x5631cbd5d230 end 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is done 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 4 starting trim on bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbd763b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbd763b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 enqueued request req=0x5631cbca3980 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 req: 0x5631cbca3980 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is io blocked 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa2fc7e8640 20 dequeued request req=0x5631cbca3980 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa2fc7e8640 20 RGWWQ: empty 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa2fc7e8640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 : miss 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa2fc7e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa2fc7e8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa2fc7e8640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 info.flags=0x17 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa2fc7e8640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 to cache LRU end 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa2fc7e8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa2fc7e8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa2fc7e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa2fc7e8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-15 : miss 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa2fc7e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa2fc7e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa2fc7e8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-15 info.flags=0x0 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa2fc7e8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-15 to cache LRU end 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa2fc7e8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-15 : miss 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa2fc7e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa2fc7e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa2fc7e8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-15 info.flags=0x0 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa2fc7e8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-15 to cache LRU end 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa2fc7e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding source pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa2fc7e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding dest pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa2fc7e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa2fc7e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa2fc7e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbd763b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbd763b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbd763b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbd763b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbcb5a60:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbd74080:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbd74080:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.102+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 2707992ef2d58685df23573385b022fbbdb2103ead9872c3fb2da4c5a89225e9 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.103+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2707992ef2d58685df23573385b022fbbdb2103ead9872c3fb2da4c5a89225e9 2026-03-20T11:45:59.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.103+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.103+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.103+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.103+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.103+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 30393fad3c4d46ac145f5be3393e0fc44b331d68dd17a52ec75efdd5bdf7f865 2026-03-20T11:45:59.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.103+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=30393fad3c4d46ac145f5be3393e0fc44b331d68dd17a52ec75efdd5bdf7f865 2026-03-20T11:45:59.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.103+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.103+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.103+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.103+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=27, curl_handle=0x5631cb8bd9e0 2026-03-20T11:45:59.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.103+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8be0f0 req_data->id=27, curl_handle=0x5631cb8bd9e0 2026-03-20T11:45:59.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.103+0000 7fa350c1d900 20 run: stack=0x5631cbcb7f60 is io blocked 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cbca4740:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cbca4740:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cb8bc590:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa350c1d900 15 stack 0x5631cbd718c0 end 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa350c1d900 20 run: stack=0x5631cbd718c0 is done 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77cf0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa350c1d900 20 collect(): s=0x5631cbc36db0 stack=0x5631cbd718c0 is complete 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa350c1d900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa350c1d900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77cf0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa350c1d900 15 stack 0x5631cbc36db0 end 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa350c1d900 20 stack->operate() returned ret=-22 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is done 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa350c1d900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa350c1d900 4 starting trim on bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbda1c60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbda1c60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa350c1d900 20 enqueued request req=0x5631cb8cc640 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa350c1d900 20 req: 0x5631cb8cc640 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is io blocked 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa2fbfe7640 20 dequeued request req=0x5631cb8cc640 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa2fbfe7640 20 RGWWQ: empty 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa2fbfe7640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 : miss 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa2fbfe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa2fbfe7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa2fbfe7640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 info.flags=0x17 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa2fbfe7640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 to cache LRU end 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa2fbfe7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa2fbfe7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa2fbfe7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa2fbfe7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-13 : miss 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa2fbfe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa2fbfe7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa2fbfe7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-13 info.flags=0x0 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa2fbfe7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-13 to cache LRU end 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa2fbfe7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-13 : miss 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa2fbfe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa2fbfe7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa2fbfe7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-13 info.flags=0x0 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa2fbfe7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-13 to cache LRU end 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa2fbfe7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18): adding source pipe: {s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa2fbfe7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18): adding dest pipe: {s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa2fbfe7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.139+0000 7fa2fbfe7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa2fbfe7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbda1c60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbda1c60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbda1c60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbda1c60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cb8bc590:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cbda02b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cbda02b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 4bdc5ba57b79a15b603866fed4df3cdc604dd3d0596fd62f6c936797df653d68 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:4bdc5ba57b79a15b603866fed4df3cdc604dd3d0596fd62f6c936797df653d68 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = b8080d6fa38816a77a423e350145bbf7dd4d8a1619d04f2b91eadd79ff535b8e 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b8080d6fa38816a77a423e350145bbf7dd4d8a1619d04f2b91eadd79ff535b8e 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=28, curl_handle=0x5631cbcad4d0 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa2a9ffb640 20 link_request req_data=0x5631cbcadbe0 req_data->id=28, curl_handle=0x5631cbcad4d0 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.140+0000 7fa350c1d900 20 run: stack=0x5631cbd718c0 is io blocked 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.141+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbca5c20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.141+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbca5c20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.141+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbca5c20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.141+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbca5c20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.141+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbda3150:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.141+0000 7fa350c1d900 15 stack 0x5631cbcabf70 end 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.141+0000 7fa350c1d900 20 run: stack=0x5631cbcabf70 is done 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.141+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda7040:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.141+0000 7fa350c1d900 20 collect(): s=0x5631cbd84c00 stack=0x5631cbcabf70 is complete 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.141+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.141+0000 7fa350c1d900 10 trimming bilogs for bucket=:dipepu-9[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14]) markers=[,00000000002.13.6,,,,,,,,,], shards=11 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.141+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbca5c20:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.141+0000 7fa350c1d900 10 trimming bilog shard 1 of :dipepu-9[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14]) at marker 00000000002.13.6 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.141+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.141+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbda3150:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.141+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbda3150:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.141+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14.1 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.141+0000 7fa350c1d900 20 run: stack=0x5631cbcabf70 is io blocked 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbda3150:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbda3150:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa350c1d900 15 stack 0x5631cbcabf70 end 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa350c1d900 20 run: stack=0x5631cbcabf70 is done 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbca5c20:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda7040:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa350c1d900 20 trim: trimmed bucket instance dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa350c1d900 15 stack 0x5631cbd84c00 end 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is done 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda7040:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa350c1d900 4 starting trim on bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbd7cc00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbd7cc00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa350c1d900 20 enqueued request req=0x5631cbc36b80 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa350c1d900 20 req: 0x5631cbc36b80 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is io blocked 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa30f7fe640 20 dequeued request req=0x5631cbc36b80 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa30f7fe640 20 RGWWQ: empty 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa30f7fe640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 : miss 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa30f7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa30f7fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa30f7fe640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 info.flags=0x17 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa30f7fe640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 to cache LRU end 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa30f7fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa30f7fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa30f7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa30f7fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-11 : miss 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa30f7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa30f7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa30f7fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-11 info.flags=0x0 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa30f7fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-11 to cache LRU end 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa30f7fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-11 : miss 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.142+0000 7fa30f7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa30f7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa30f7fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-11 info.flags=0x0 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa30f7fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-11 to cache LRU end 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa30f7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16): adding source pipe: {s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbca54a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa30f7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16): adding dest pipe: {s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa30f7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa30f7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa30f7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbca54a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbd7cc00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbca54a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbd7cc00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbca54a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbd7cc00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cb8cd520:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 15 stack 0x5631cbca8810 end 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 run: stack=0x5631cbca8810 is done 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbd7cc00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 collect(): s=0x5631cbcb6b00 stack=0x5631cbca8810 is complete 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 10 trimming bilogs for bucket=:dipepu-38[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43]) markers=[,,,,,,,,00000000001.38.6,,], shards=11 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda7040:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbca5c20:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 10 trimming bilog shard 8 of :dipepu-38[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43]) at marker 00000000001.38.6 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd7f210:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda7040:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbda3150:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd87cb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbda3150:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43.8 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 run: stack=0x5631cbcabf70 is io blocked 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd87cb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 3edb5d2ca4d4e17de0a1adcb61bf64654ca007829f3f7c9efc9c6a3b73819571 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:3edb5d2ca4d4e17de0a1adcb61bf64654ca007829f3f7c9efc9c6a3b73819571 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = cba9cb2c80f95bffa811531d7b73aeec8fec7f4242703e4cd1451fb1d5f73eee 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=cba9cb2c80f95bffa811531d7b73aeec8fec7f4242703e4cd1451fb1d5f73eee 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=29, curl_handle=0x5631cb8cf450 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8cff10 req_data->id=29, curl_handle=0x5631cb8cf450 2026-03-20T11:45:59.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.143+0000 7fa350c1d900 20 run: stack=0x5631cbca8810 is io blocked 2026-03-20T11:45:59.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.144+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbda3150:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.144+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbda3150:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.144+0000 7fa350c1d900 15 stack 0x5631cbcabf70 end 2026-03-20T11:45:59.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.144+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.144+0000 7fa350c1d900 20 run: stack=0x5631cbcabf70 is done 2026-03-20T11:45:59.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.144+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbca5c20:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.144+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.144+0000 7fa350c1d900 20 trim: trimmed bucket instance dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:45:59.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.144+0000 7fa350c1d900 15 stack 0x5631cbcb6b00 end 2026-03-20T11:45:59.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.144+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is done 2026-03-20T11:45:59.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.144+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.144+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.144+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.144+0000 7fa350c1d900 4 starting trim on bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:45:59.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.144+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbda1c60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.144+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbda1c60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.144+0000 7fa350c1d900 20 enqueued request req=0x5631cbd71a80 2026-03-20T11:45:59.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.144+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.144+0000 7fa350c1d900 20 req: 0x5631cbd71a80 2026-03-20T11:45:59.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.144+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is io blocked 2026-03-20T11:45:59.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.144+0000 7fa2fcfe9640 20 dequeued request req=0x5631cbd71a80 2026-03-20T11:45:59.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.144+0000 7fa2fcfe9640 20 RGWWQ: empty 2026-03-20T11:45:59.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.144+0000 7fa2fcfe9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 : miss 2026-03-20T11:45:59.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.144+0000 7fa2fcfe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbd74080:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa2fcfe9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa2fcfe9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 info.flags=0x17 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa2fcfe9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 to cache LRU end 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa2fcfe9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa2fcfe9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa2fcfe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa2fcfe9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-12 : miss 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa2fcfe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbd74080:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbd74080:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbd74080:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbcb5a60:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 15 stack 0x5631cbcb7f60 end 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 run: stack=0x5631cbcb7f60 is done 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 collect(): s=0x5631cbd5d230 stack=0x5631cbcb7f60 is complete 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 10 trimming bilogs for bucket=:dipepu-15[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20]) markers=[,,,,,,,,00000000007.20.6,,], shards=11 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbcb5a60:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 10 trimming bilog shard 8 of :dipepu-15[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20]) at marker 00000000007.20.6 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbda3150:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbda3150:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20.8 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 run: stack=0x5631cbcb7f60 is io blocked 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa2fcfe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa2fcfe9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-12 info.flags=0x0 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa2fcfe9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-12 to cache LRU end 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa2fcfe9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-12 : miss 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa2fcfe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa2fcfe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa2fcfe9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-12 info.flags=0x0 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa2fcfe9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-12 to cache LRU end 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa2fcfe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17): adding source pipe: {s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa2fcfe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17): adding dest pipe: {s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa2fcfe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa2fcfe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa2fcfe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbda1c60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbda1c60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbda1c60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbda1c60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbda1c60:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbda99c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbda99c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.144 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 51cf4ab9ad88bc62787efe50985a89a41eb8dfe66fb5a4faf213186535ae0f74 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:51cf4ab9ad88bc62787efe50985a89a41eb8dfe66fb5a4faf213186535ae0f74 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = dc3e21c64aef0586a117f34410659a24aa328759b73537ad22dd2fb339241ef4 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=dc3e21c64aef0586a117f34410659a24aa328759b73537ad22dd2fb339241ef4 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=30, curl_handle=0x5631cb8bd9e0 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8be0f0 req_data->id=30, curl_handle=0x5631cb8bd9e0 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.145+0000 7fa350c1d900 20 run: stack=0x5631cbcabf70 is io blocked 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbda3150:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbda3150:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa350c1d900 15 stack 0x5631cbcb7f60 end 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa350c1d900 20 run: stack=0x5631cbcb7f60 is done 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbcb5a60:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa350c1d900 20 trim: trimmed bucket instance dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa350c1d900 15 stack 0x5631cbd5d230 end 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is done 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa350c1d900 4 starting trim on bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbcb5a60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbcb5a60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa350c1d900 20 enqueued request req=0x5631cbca3980 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa350c1d900 20 req: 0x5631cbca3980 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is io blocked 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa2fb7e6640 20 dequeued request req=0x5631cbca3980 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa2fb7e6640 20 RGWWQ: empty 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa2fb7e6640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 : miss 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa2fb7e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa2fb7e6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:59.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa2fb7e6640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 info.flags=0x17 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa2fb7e6640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 to cache LRU end 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa2fb7e6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa2fb7e6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa2fb7e6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa2fb7e6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-40 : miss 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.146+0000 7fa2fb7e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa2fb7e6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa2fb7e6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-40 info.flags=0x0 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa2fb7e6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-40 to cache LRU end 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa2fb7e6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-40 : miss 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa2fb7e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa2fb7e6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa2fb7e6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-40 info.flags=0x0 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa2fb7e6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-40 to cache LRU end 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa2fb7e6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1): adding source pipe: {s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa2fb7e6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1): adding dest pipe: {s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa2fb7e6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa2fb7e6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa2fb7e6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbcb5a60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbcb5a60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbcb5a60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbcb5a60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbdb3000:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbcb5a60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbcb5a60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.146 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.147 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 13515a175dd174c65180db34523cdc2e412e7eed81faa5e32fd8655d7d7af0a8 2026-03-20T11:45:59.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.147 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.147 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.147 INFO:teuthology.orchestra.run.vm02.stderr:13515a175dd174c65180db34523cdc2e412e7eed81faa5e32fd8655d7d7af0a8 2026-03-20T11:45:59.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 83e9deac7fe5fc2c4ce9317153039aed4535ad00556863030cccdc4d1d63b6e1 2026-03-20T11:45:59.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=83e9deac7fe5fc2c4ce9317153039aed4535ad00556863030cccdc4d1d63b6e1 2026-03-20T11:45:59.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=31, curl_handle=0x5631cb8c1990 2026-03-20T11:45:59.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8c20a0 req_data->id=31, curl_handle=0x5631cb8c1990 2026-03-20T11:45:59.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.147+0000 7fa350c1d900 20 run: stack=0x5631cbcb7f60 is io blocked 2026-03-20T11:45:59.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.182+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cbda02b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.182+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cbda02b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.182+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cbda02b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.182+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cbda02b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.182+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cb8bc590:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.182+0000 7fa350c1d900 15 stack 0x5631cbd718c0 end 2026-03-20T11:45:59.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.182+0000 7fa350c1d900 20 run: stack=0x5631cbd718c0 is done 2026-03-20T11:45:59.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.182+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.182+0000 7fa350c1d900 20 collect(): s=0x5631cbc36db0 stack=0x5631cbd718c0 is complete 2026-03-20T11:45:59.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.182+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.182+0000 7fa350c1d900 10 trimming bilogs for bucket=:dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) markers=[,,,,,00000000003.29.10,,,00000000003.11.6,00000000011.37.6,], shards=11 2026-03-20T11:45:59.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.182+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdaa6f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.182+0000 7fa350c1d900 10 trimming bilog shard 5 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000003.29.10 2026-03-20T11:45:59.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.182+0000 7fa350c1d900 10 trimming bilog shard 8 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000003.11.6 2026-03-20T11:45:59.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.182+0000 7fa350c1d900 10 trimming bilog shard 9 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000011.37.6 2026-03-20T11:45:59.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.182+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.182+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cbca4740:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.182+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd86f70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.182+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbda8810:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.182+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cbca4740:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.182+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.5 2026-03-20T11:45:59.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.182+0000 7fa350c1d900 20 run: stack=0x5631cbd718c0 is io blocked 2026-03-20T11:45:59.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.182+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd86f70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.182+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.8 2026-03-20T11:45:59.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.182+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is io blocked 2026-03-20T11:45:59.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.182+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbda8810:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.182+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.9 2026-03-20T11:45:59.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.182+0000 7fa350c1d900 20 run: stack=0x5631cbd72b00 is io blocked 2026-03-20T11:45:59.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cbca4740:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cbca4740:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 15 stack 0x5631cbd718c0 end 2026-03-20T11:45:59.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 20 run: stack=0x5631cbd718c0 is done 2026-03-20T11:45:59.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdaa6f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd86f70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd86f70:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 15 stack 0x5631cbca2ac0 end 2026-03-20T11:45:59.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is done 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbda8810:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbda8810:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 15 stack 0x5631cbd72b00 end 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 20 run: stack=0x5631cbd72b00 is done 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdaa6f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdaa6f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 20 trim: trimmed bucket instance dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 15 stack 0x5631cbc36db0 end 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is done 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 4 starting trim on bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbda3150:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbda3150:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 20 enqueued request req=0x5631cbcac160 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 20 req: 0x5631cbcac160 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.183+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is io blocked 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa30e7fc640 20 dequeued request req=0x5631cbcac160 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa30e7fc640 20 RGWWQ: empty 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa30e7fc640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 : miss 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa30e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa30e7fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa30e7fc640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 info.flags=0x17 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa30e7fc640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 to cache LRU end 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa30e7fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa30e7fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa30e7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa30e7fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-18 : miss 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa30e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa30e7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa30e7fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-18 info.flags=0x0 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa30e7fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-18 to cache LRU end 2026-03-20T11:45:59.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa30e7fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-18 : miss 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa30e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa30e7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa30e7fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-18 info.flags=0x0 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa30e7fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-18 to cache LRU end 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa30e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23): adding source pipe: {s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa30e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23): adding dest pipe: {s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa30e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa30e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa30e7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbda3150:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbda3150:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbda3150:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbda3150:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cb8bc590:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbda02b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbda02b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = c8d939a92f24c5e831521602e98ab34005a25dacdcea9d7fb28bf4f15e5dc4bc 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:c8d939a92f24c5e831521602e98ab34005a25dacdcea9d7fb28bf4f15e5dc4bc 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 9a10ad9722a1ebb10ce08410b3f80f730f12f94020188724572d0ea71b1624f9 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=9a10ad9722a1ebb10ce08410b3f80f730f12f94020188724572d0ea71b1624f9 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=32, curl_handle=0x5631cbcad4d0 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa2a9ffb640 20 link_request req_data=0x5631cbcadbe0 req_data->id=32, curl_handle=0x5631cbcad4d0 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.184+0000 7fa350c1d900 20 run: stack=0x5631cbd72b00 is io blocked 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd87cb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd87cb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd87cb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd87cb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd7f210:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 15 stack 0x5631cbca8810 end 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 run: stack=0x5631cbca8810 is done 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda7040:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 collect(): s=0x5631cbd84c00 stack=0x5631cbca8810 is complete 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 10 trimming bilogs for bucket=:dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) markers=[00000000014.52.6,00000000008.34.6,00000000008.50.6,00000000008.32.6,00000000012.28.6,00000000010.31.6,00000000008.21.6,00000000010.27.6,00000000004.27.6,00000000008.22.6,00000000008.19.6], shards=11 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbdaa6f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 10 trimming bilog shard 0 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000014.52.6 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 10 trimming bilog shard 1 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.34.6 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 10 trimming bilog shard 2 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.50.6 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 10 trimming bilog shard 3 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.32.6 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 10 trimming bilog shard 4 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000012.28.6 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 10 trimming bilog shard 5 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000010.31.6 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 10 trimming bilog shard 6 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.21.6 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 10 trimming bilog shard 7 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000010.27.6 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 10 trimming bilog shard 8 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000004.27.6 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 10 trimming bilog shard 9 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.22.6 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 10 trimming bilog shard 10 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.19.6 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd87ad0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd7bfe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cbd7cd40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbcb6090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cbca6d40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cb8cca80:op=0x5631cbda49d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cbdaae70:op=0x5631cbdac8b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cb8c02a0:op=0x5631cbdab220:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdaba20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cbd9ef90:op=0x5631cbdae040:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cbca6250:op=0x5631cbdae840:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd87ad0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.0 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 run: stack=0x5631cbca8810 is io blocked 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd7bfe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.1 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is io blocked 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cbd7cd40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.2 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 run: stack=0x5631cbd718c0 is io blocked 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbcb6090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.3 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 run: stack=0x5631cbd79340 is io blocked 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cbca6d40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.4 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 run: stack=0x5631cbca9120 is io blocked 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cb8cca80:op=0x5631cbda49d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.5 2026-03-20T11:45:59.183 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 run: stack=0x5631cb8cca80 is io blocked 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cbdaae70:op=0x5631cbdac8b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.6 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 run: stack=0x5631cbdaae70 is io blocked 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cb8c02a0:op=0x5631cbdab220:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.7 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 run: stack=0x5631cb8c02a0 is io blocked 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdaba20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.8 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 run: stack=0x5631cb8cdc90 is io blocked 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cbd9ef90:op=0x5631cbdae040:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.9 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 run: stack=0x5631cbd9ef90 is io blocked 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 cr:s=0x5631cbca6250:op=0x5631cbdae840:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.10 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.185+0000 7fa350c1d900 20 run: stack=0x5631cbca6250 is io blocked 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.186+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd87ad0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.186+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd87ad0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.186+0000 7fa350c1d900 15 stack 0x5631cbca8810 end 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.186+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.186+0000 7fa350c1d900 20 run: stack=0x5631cbca8810 is done 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.186+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbdaa6f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.186+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cbca6d40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cbca6d40:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 15 stack 0x5631cbca9120 end 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 run: stack=0x5631cbca9120 is done 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbdaa6f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cbca6250:op=0x5631cbdae840:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cbca6250:op=0x5631cbdae840:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 15 stack 0x5631cbca6250 end 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 run: stack=0x5631cbca6250 is done 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbdaa6f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cb8cca80:op=0x5631cbda49d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cb8cca80:op=0x5631cbda49d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 15 stack 0x5631cb8cca80 end 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 run: stack=0x5631cb8cca80 is done 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbdaa6f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cb8c02a0:op=0x5631cbdab220:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cb8c02a0:op=0x5631cbdab220:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 15 stack 0x5631cb8c02a0 end 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 run: stack=0x5631cb8c02a0 is done 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbdaa6f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cbd9ef90:op=0x5631cbdae040:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cbd9ef90:op=0x5631cbdae040:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 15 stack 0x5631cbd9ef90 end 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 run: stack=0x5631cbd9ef90 is done 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbdaa6f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbcb6090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbcb6090:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 15 stack 0x5631cbd79340 end 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 run: stack=0x5631cbd79340 is done 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbdaa6f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdaba20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdaba20:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 15 stack 0x5631cb8cdc90 end 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 run: stack=0x5631cb8cdc90 is done 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbdaa6f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cbd7cd40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cbd718c0:op=0x5631cbd7cd40:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 15 stack 0x5631cbd718c0 end 2026-03-20T11:45:59.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 run: stack=0x5631cbd718c0 is done 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbdaa6f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd7bfe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd7bfe0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 15 stack 0x5631cbca2ac0 end 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is done 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbdaa6f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cbdaae70:op=0x5631cbdac8b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cbdaae70:op=0x5631cbdac8b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 15 stack 0x5631cbdaae70 end 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 run: stack=0x5631cbdaae70 is done 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbdaa6f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda7040:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 trim: trimmed bucket instance dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 15 stack 0x5631cbd84c00 end 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is done 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdab220:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 4 starting trim on bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbda49d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbda49d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 enqueued request req=0x5631cb8cc640 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 req: 0x5631cb8cc640 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa350c1d900 20 run: stack=0x5631cb8cdc90 is io blocked 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa30effd640 20 dequeued request req=0x5631cb8cc640 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa30effd640 20 RGWWQ: empty 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa30effd640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 : miss 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.187+0000 7fa30effd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa30effd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa30effd640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 info.flags=0x17 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa30effd640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 to cache LRU end 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa30effd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa30effd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa30effd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa30effd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-50 : miss 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa30effd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa30effd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa30effd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-50 info.flags=0x0 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa30effd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-50 to cache LRU end 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa30effd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-50 : miss 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa30effd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa30effd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa30effd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-50 info.flags=0x0 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa30effd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-50 to cache LRU end 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa30effd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10): adding source pipe: {s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa30effd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10): adding dest pipe: {s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa30effd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa30effd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa30effd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbda49d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbda49d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbda49d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbda49d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdab220:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbda2ac0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdab220:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 20 run: stack=0x5631cb8cdc90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbda49d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbda49d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.187 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 6ed966859db4596d0babe5470459bcc44f362d9abbf99454f35e375fea61d9be 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:6ed966859db4596d0babe5470459bcc44f362d9abbf99454f35e375fea61d9be 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = abff12076f881d8a3a5858ede5a0ff3327eb91ca8e60bcfdd5f4490a5b4cf150 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=abff12076f881d8a3a5858ede5a0ff3327eb91ca8e60bcfdd5f4490a5b4cf150 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=33, curl_handle=0x5631cb8cf450 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8cff10 req_data->id=33, curl_handle=0x5631cb8cf450 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.188+0000 7fa350c1d900 20 run: stack=0x5631cbd79340 is io blocked 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbda99c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbda99c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbcb5a60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbda99c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbcb5a60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbda99c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbcb5a60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbda1c60:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 15 stack 0x5631cbcabf70 end 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 run: stack=0x5631cbcabf70 is done 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbcb5a60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 collect(): s=0x5631cbcb6b00 stack=0x5631cbcabf70 is complete 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 10 trimming bilogs for bucket=:dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) markers=[00000000014.44.6,00000000008.52.6,00000000008.40.6,00000000008.21.6,00000000012.26.6,00000000010.26.6,00000000008.35.6,00000000010.32.6,00000000004.38.6,00000000008.22.6,00000000008.23.6], shards=11 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbdb3000:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 15 stack 0x5631cbcb7f60 end 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 run: stack=0x5631cbcb7f60 is done 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbca5c20:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 10 trimming bilog shard 0 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000014.44.6 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 10 trimming bilog shard 1 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.52.6 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 10 trimming bilog shard 2 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.40.6 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 10 trimming bilog shard 3 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.21.6 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 10 trimming bilog shard 4 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000012.26.6 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 10 trimming bilog shard 5 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000010.26.6 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 10 trimming bilog shard 6 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.35.6 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 10 trimming bilog shard 7 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000010.32.6 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 10 trimming bilog shard 8 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000004.38.6 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 10 trimming bilog shard 9 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.22.6 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 10 trimming bilog shard 10 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.23.6 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 collect(): s=0x5631cbd5d230 stack=0x5631cbcb7f60 is complete 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 10 trimming bilogs for bucket=:dipepu-40[4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1]) markers=[,,,,,00000000001.103.6,,,,,], shards=11 2026-03-20T11:45:59.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbd85300:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbd9ef90:op=0x5631cbda99c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cb8c02a0:op=0x5631cbd87240:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbca6250:op=0x5631cbd87ad0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cbd73d10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbda1a10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbd7cd40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbd882d0:op=0x5631cbdae080:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbdac8b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbdad0b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda67a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbcb5a60:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 10 trimming bilog shard 5 of :dipepu-40[4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1]) at marker 00000000001.103.6 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbd85300:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.0 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 run: stack=0x5631cbcabf70 is io blocked 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbd9ef90:op=0x5631cbda99c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.1 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 run: stack=0x5631cbd9ef90 is io blocked 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cb8c02a0:op=0x5631cbd87240:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.2 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 run: stack=0x5631cb8c02a0 is io blocked 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbca6250:op=0x5631cbd87ad0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.3 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 run: stack=0x5631cbca6250 is io blocked 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cbd73d10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.4 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 run: stack=0x5631cbca9120 is io blocked 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbda1a10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.5 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 run: stack=0x5631cbca8810 is io blocked 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbd7cd40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.6 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 run: stack=0x5631cbd731d0 is io blocked 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbd882d0:op=0x5631cbdae080:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.7 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 run: stack=0x5631cbd882d0 is io blocked 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbdac8b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.8 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.189+0000 7fa350c1d900 20 run: stack=0x5631cbda4490 is io blocked 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbdad0b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.9 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is io blocked 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda67a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.10 2026-03-20T11:45:59.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is io blocked 2026-03-20T11:45:59.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbda6fa0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 cr:s=0x5631cbca6250:op=0x5631cbd87ad0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 cr:s=0x5631cbca6250:op=0x5631cbd87ad0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 15 stack 0x5631cbca6250 end 2026-03-20T11:45:59.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 run: stack=0x5631cbca6250 is done 2026-03-20T11:45:59.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbda6fa0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1.5 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 run: stack=0x5631cbcb7f60 is io blocked 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbca5c20:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 cr:s=0x5631cbd882d0:op=0x5631cbdae080:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 cr:s=0x5631cbd882d0:op=0x5631cbdae080:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 15 stack 0x5631cbd882d0 end 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 run: stack=0x5631cbd882d0 is done 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cbd73d10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cbd73d10:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 15 stack 0x5631cbca9120 end 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 run: stack=0x5631cbca9120 is done 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbda1a10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbda1a10:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 15 stack 0x5631cbca8810 end 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 run: stack=0x5631cbca8810 is done 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbca5c20:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbca5c20:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbca5c20:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.190+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.191+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbdad0b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.191+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbdad0b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.191+0000 7fa350c1d900 15 stack 0x5631cbca2ac0 end 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.191+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.191+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is done 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.191+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbca5c20:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.191+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.191+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda67a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.191+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda67a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.191+0000 7fa350c1d900 15 stack 0x5631cbd84c00 end 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.191+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.191+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is done 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.191+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbca5c20:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.191+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbdac8b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbdac8b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 15 stack 0x5631cbda4490 end 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 run: stack=0x5631cbda4490 is done 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbca5c20:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbd85300:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 cr:s=0x5631cbcabf70:op=0x5631cbd85300:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 15 stack 0x5631cbcabf70 end 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 run: stack=0x5631cbcabf70 is done 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbca5c20:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbda6fa0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 cr:s=0x5631cbcb7f60:op=0x5631cbda6fa0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 15 stack 0x5631cbcb7f60 end 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 run: stack=0x5631cbcb7f60 is done 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbcb5a60:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 cr:s=0x5631cb8c02a0:op=0x5631cbd87240:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 cr:s=0x5631cb8c02a0:op=0x5631cbd87240:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 15 stack 0x5631cb8c02a0 end 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 run: stack=0x5631cb8c02a0 is done 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 cr:s=0x5631cbd5d230:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 trim: trimmed bucket instance dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 15 stack 0x5631cbd5d230 end 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 run: stack=0x5631cbd5d230 is done 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbca5c20:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 4 starting trim on bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbdb2f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbdb2f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 enqueued request req=0x5631cbd5d040 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 req: 0x5631cbd5d040 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 run: stack=0x5631cbda4490 is io blocked 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30d7fa640 20 dequeued request req=0x5631cbd5d040 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30d7fa640 20 RGWWQ: empty 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30d7fa640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 : miss 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbd7cd40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbd7cd40:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 15 stack 0x5631cbd731d0 end 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 run: stack=0x5631cbd731d0 is done 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbca5c20:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 cr:s=0x5631cbd9ef90:op=0x5631cbda99c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 cr:s=0x5631cbd9ef90:op=0x5631cbda99c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 15 stack 0x5631cbd9ef90 end 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 run: stack=0x5631cbd9ef90 is done 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbca5c20:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 cr:s=0x5631cbcb6b00:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 trim: trimmed bucket instance dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 15 stack 0x5631cbcb6b00 end 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 run: stack=0x5631cbcb6b00 is done 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 4 starting trim on bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbda99c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbda99c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 enqueued request req=0x5631cbca78e0 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 req: 0x5631cbca78e0 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa350c1d900 20 run: stack=0x5631cbd731d0 is io blocked 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30dffb640 20 dequeued request req=0x5631cbca78e0 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30dffb640 20 RGWWQ: empty 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30dffb640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 : miss 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30d7fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30d7fa640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 info.flags=0x17 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30d7fa640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 to cache LRU end 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30d7fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30d7fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30d7fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30d7fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-24 : miss 2026-03-20T11:45:59.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30dffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30dffb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 info.flags=0x17 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30dffb640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 to cache LRU end 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30dffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30dffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30dffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30dffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-56 : miss 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30d7fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30d7fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-24 info.flags=0x0 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30d7fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-24 to cache LRU end 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30d7fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-24 : miss 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30dffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.192+0000 7fa30dffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-56 info.flags=0x0 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa30dffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-56 to cache LRU end 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa30dffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-56 : miss 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa30dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa30d7fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa30d7fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-24 info.flags=0x0 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa30d7fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-24 to cache LRU end 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa30d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29): adding source pipe: {s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa30d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29): adding dest pipe: {s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa30d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa30d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa30d7fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbdb2f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbdb2f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbdb2f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbdb2f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cb8ce9e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 run: stack=0x5631cbda4490 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbcb5a60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbcb5a60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.193 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 15cb57426dddf4cd578097571ac6c1dfac5c30a306cb48285dcb0b18645805a3 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:15cb57426dddf4cd578097571ac6c1dfac5c30a306cb48285dcb0b18645805a3 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = c00ba2b01dc8c649a4b5ea9c3f45421d303317aac58596a05260a27e24a8c1cc 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c00ba2b01dc8c649a4b5ea9c3f45421d303317aac58596a05260a27e24a8c1cc 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=34, curl_handle=0x5631cb8c1990 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8c20a0 req_data->id=34, curl_handle=0x5631cb8c1990 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is io blocked 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa30dffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa30dffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-56 info.flags=0x0 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa30dffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-56 to cache LRU end 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa30dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16): adding source pipe: {s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa30dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16): adding dest pipe: {s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa30dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa30dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa30dffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbda99c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbda99c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbda99c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbda99c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd7f210:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 run: stack=0x5631cbd731d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbdaa6f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbdaa6f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.194 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.195 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.195 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.195 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.195 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.195 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.195 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.195 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 5b056f72a7292c130821f95e11244d1f96d76e97a9ad6b81f418c315b1c10835 2026-03-20T11:45:59.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.195 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.195 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.195 INFO:teuthology.orchestra.run.vm02.stderr:5b056f72a7292c130821f95e11244d1f96d76e97a9ad6b81f418c315b1c10835 2026-03-20T11:45:59.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 2681efd305de9214a943b61a5efee1e25b3f8dc64333767ba1bb268800d22066 2026-03-20T11:45:59.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2681efd305de9214a943b61a5efee1e25b3f8dc64333767ba1bb268800d22066 2026-03-20T11:45:59.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=35, curl_handle=0x5631cb8bd9e0 2026-03-20T11:45:59.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8be0f0 req_data->id=35, curl_handle=0x5631cb8bd9e0 2026-03-20T11:45:59.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.193+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is io blocked 2026-03-20T11:45:59.195 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:59.192+0000 7fcff2232640 -1 req 13925657171621669633 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:59.195 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:59.192+0000 7fcff3234640 -1 req 2821805877975283079 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:59.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.228+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbda02b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.228+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbda02b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.228+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbda02b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.228+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbda02b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.228+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cb8bc590:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.228+0000 7fa350c1d900 15 stack 0x5631cbd72b00 end 2026-03-20T11:45:59.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.228+0000 7fa350c1d900 20 run: stack=0x5631cbd72b00 is done 2026-03-20T11:45:59.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.228+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.228+0000 7fa350c1d900 20 collect(): s=0x5631cbc36db0 stack=0x5631cbd72b00 is complete 2026-03-20T11:45:59.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.228+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.228+0000 7fa350c1d900 10 trimming bilogs for bucket=:dipepu-18[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23]) markers=[,,,,,,,,00000000005.51.12,,], shards=11 2026-03-20T11:45:59.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.228+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbca5c20:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.228+0000 7fa350c1d900 10 trimming bilog shard 8 of :dipepu-18[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23]) at marker 00000000005.51.12 2026-03-20T11:45:59.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.228+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.228+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbda99c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.228+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbda99c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.228+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23.8 2026-03-20T11:45:59.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.228+0000 7fa350c1d900 20 run: stack=0x5631cbd72b00 is io blocked 2026-03-20T11:45:59.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbda99c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbda99c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa350c1d900 15 stack 0x5631cbd72b00 end 2026-03-20T11:45:59.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa350c1d900 20 run: stack=0x5631cbd72b00 is done 2026-03-20T11:45:59.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbca5c20:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa350c1d900 20 trim: trimmed bucket instance dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:45:59.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa350c1d900 15 stack 0x5631cbc36db0 end 2026-03-20T11:45:59.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is done 2026-03-20T11:45:59.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa350c1d900 4 starting trim on bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:45:59.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdac880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdac880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa350c1d900 20 enqueued request req=0x5631cbcac160 2026-03-20T11:45:59.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa350c1d900 20 req: 0x5631cbcac160 2026-03-20T11:45:59.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is io blocked 2026-03-20T11:45:59.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa30cff9640 20 dequeued request req=0x5631cbcac160 2026-03-20T11:45:59.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa30cff9640 20 RGWWQ: empty 2026-03-20T11:45:59.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa30cff9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 : miss 2026-03-20T11:45:59.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa30cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa30cff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:59.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa30cff9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 info.flags=0x17 2026-03-20T11:45:59.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa30cff9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 to cache LRU end 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa30cff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa30cff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa30cff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa30cff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-27 : miss 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.229+0000 7fa30cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbda49d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa30cff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa30cff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-27 info.flags=0x0 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa30cff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-27 to cache LRU end 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa30cff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-27 : miss 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa30cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbda49d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbda49d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbda49d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbda2ac0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 15 stack 0x5631cbd79340 end 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 run: stack=0x5631cbd79340 is done 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdab220:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 collect(): s=0x5631cb8cdc90 stack=0x5631cbd79340 is complete 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdab220:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 15 stack 0x5631cb8cdc90 end 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 stack->operate() returned ret=-22 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 run: stack=0x5631cb8cdc90 is done 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdab220:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 4 starting trim on bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbda99c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbda99c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 enqueued request req=0x5631cb8cc640 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 req: 0x5631cb8cc640 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 run: stack=0x5631cb8cdc90 is io blocked 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa307fff640 20 dequeued request req=0x5631cb8cc640 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa307fff640 20 RGWWQ: empty 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa307fff640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 : miss 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa307fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa30cff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa30cff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-27 info.flags=0x0 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa30cff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-27 to cache LRU end 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa30cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32): adding source pipe: {s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa30cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32): adding dest pipe: {s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa30cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa30cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa30cff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdac880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdac880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdac880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdac880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa307fff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=314 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbda02b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa307fff640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 info.flags=0x17 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbd87c90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbd87c90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa307fff640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 to cache LRU end 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa307fff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa307fff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa307fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa307fff640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-44 : miss 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa307fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 8abfce4b5cc0fa4babab6dbf305ea434a09c962c9ebe05e9835e96c6ae7ef8d3 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:8abfce4b5cc0fa4babab6dbf305ea434a09c962c9ebe05e9835e96c6ae7ef8d3 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = ae4dfeeb1d9729200f3cbc8b0bb31f56e2192ecc958ea8f3fa7ab4ad43c91bd6 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=ae4dfeeb1d9729200f3cbc8b0bb31f56e2192ecc958ea8f3fa7ab4ad43c91bd6 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=36, curl_handle=0x5631cb8cf450 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8cff10 req_data->id=36, curl_handle=0x5631cb8cf450 2026-03-20T11:45:59.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa350c1d900 20 run: stack=0x5631cbd79340 is io blocked 2026-03-20T11:45:59.229 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:59.230+0000 7fcfeba25640 -1 req 4794568595841448240 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:59.229 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:59.230+0000 7fd070b2f640 -1 req 14280632032180330712 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa307fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa307fff640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-44 info.flags=0x0 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa307fff640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-44 to cache LRU end 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa307fff640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-44 : miss 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.230+0000 7fa307fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa307fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa307fff640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-44 info.flags=0x0 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa307fff640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-44 to cache LRU end 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa307fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4): adding source pipe: {s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa307fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4): adding dest pipe: {s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa307fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa307fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa307fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbda99c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbda99c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbda99c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbda99c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdab220:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbca5c20:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdab220:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 20 run: stack=0x5631cb8cdc90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cb8bc590:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cb8bc590:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.229 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 990ee6396f3c18b6f61efebbafd7fe6d5a94de279fe3aafa87f1db1866b8583e 2026-03-20T11:45:59.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.230 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.230 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.230 INFO:teuthology.orchestra.run.vm02.stderr:990ee6396f3c18b6f61efebbafd7fe6d5a94de279fe3aafa87f1db1866b8583e 2026-03-20T11:45:59.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 1d5a05682840402c20789e1e5b2830cb733af67327586ee03bde8aa85a14e32d 2026-03-20T11:45:59.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=1d5a05682840402c20789e1e5b2830cb733af67327586ee03bde8aa85a14e32d 2026-03-20T11:45:59.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=37, curl_handle=0x5631cbcad4d0 2026-03-20T11:45:59.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa2a9ffb640 20 link_request req_data=0x5631cbcadbe0 req_data->id=37, curl_handle=0x5631cbcad4d0 2026-03-20T11:45:59.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.231+0000 7fa350c1d900 20 run: stack=0x5631cbd72b00 is io blocked 2026-03-20T11:45:59.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbcb5a60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbcb5a60:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cb8ce9e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 15 stack 0x5631cbd84c00 end 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is done 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbdaa6f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbdaa6f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 collect(): s=0x5631cbda4490 stack=0x5631cbd84c00 is complete 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 15 stack 0x5631cbda4490 end 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 stack->operate() returned ret=-22 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 run: stack=0x5631cbda4490 is done 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd7f210:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 15 stack 0x5631cbca2ac0 end 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is done 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 collect(): s=0x5631cbd731d0 stack=0x5631cbca2ac0 is complete 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 15 stack 0x5631cbd731d0 end 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 stack->operate() returned ret=-22 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 run: stack=0x5631cbd731d0 is done 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 4 starting trim on bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbdad280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 4 starting trim on bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbdad280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 enqueued request req=0x5631cbd72080 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 req: 0x5631cbd72080 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 run: stack=0x5631cbda4490 is io blocked 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbdac800:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbdac800:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 enqueued request req=0x5631cbcad7c0 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 req: 0x5631cbd72080 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 req: 0x5631cbcad7c0 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa350c1d900 20 run: stack=0x5631cbd731d0 is io blocked 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa306ffd640 20 dequeued request req=0x5631cbd72080 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa306ffd640 20 RGWWQ: 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa306ffd640 20 req: 0x5631cbcad7c0 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa306ffd640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 : miss 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa306ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa3077fe640 20 dequeued request req=0x5631cbcad7c0 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa3077fe640 20 RGWWQ: empty 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa3077fe640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 : miss 2026-03-20T11:45:59.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa3077fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa306ffd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa306ffd640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 info.flags=0x17 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa306ffd640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 to cache LRU end 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa306ffd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa306ffd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa306ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa306ffd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-17 : miss 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa306ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa3077fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa3077fe640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 info.flags=0x17 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa3077fe640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 to cache LRU end 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa3077fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa3077fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa3077fe640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa3077fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa3077fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-46 : miss 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa3077fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa3077fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa3077fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-46 info.flags=0x0 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa3077fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-46 to cache LRU end 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa3077fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-46 : miss 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa3077fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa306ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa306ffd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-17 info.flags=0x0 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa306ffd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-17 to cache LRU end 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa306ffd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-17 : miss 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.234+0000 7fa306ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa306ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa306ffd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-17 info.flags=0x0 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa306ffd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-17 to cache LRU end 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa306ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22): adding source pipe: {s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa306ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22): adding dest pipe: {s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa306ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa306ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa306ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa3077fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa3077fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-46 info.flags=0x0 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa3077fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-46 to cache LRU end 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbdad280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbdad280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbdad280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa3077fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6): adding source pipe: {s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbdad280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa3077fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6): adding dest pipe: {s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd7f210:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 run: stack=0x5631cbda4490 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbcb5a60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbcb5a60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa3077fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa3077fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa3077fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 19d643874f3f42dde6e29944bafb21ab38ea33f58c2e27f8d1233d4a00f22030 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:19d643874f3f42dde6e29944bafb21ab38ea33f58c2e27f8d1233d4a00f22030 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 59301450e76c250f9b1fff1472e81b955dd6c7cb351240568a738329cc888b5a 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=59301450e76c250f9b1fff1472e81b955dd6c7cb351240568a738329cc888b5a 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=38, curl_handle=0x5631cb8bd9e0 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8be0f0 req_data->id=38, curl_handle=0x5631cb8bd9e0 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is io blocked 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbdac800:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbdac800:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbdac800:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbdac800:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cb8ce9e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 run: stack=0x5631cbd731d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbd71620:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbd71620:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.233 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = ab474b65535a78838daf567deee1e084bcdd65a2989aac45a0e443d139136f3e 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr:ab474b65535a78838daf567deee1e084bcdd65a2989aac45a0e443d139136f3e 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 09000702995de047ebb19c799c7a6947252a10b65a6a3a69e6e0e73deb6e90ae 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=09000702995de047ebb19c799c7a6947252a10b65a6a3a69e6e0e73deb6e90ae 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=39, curl_handle=0x5631cb8c1990 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8c20a0 req_data->id=39, curl_handle=0x5631cb8c1990 2026-03-20T11:45:59.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.235+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is io blocked 2026-03-20T11:45:59.234 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:59.234+0000 7fcfdaa03640 -1 req 7579396747266773176 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:59.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbd87c90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbd87c90:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:59.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbda02b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 15 stack 0x5631cbd79340 end 2026-03-20T11:45:59.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 run: stack=0x5631cbd79340 is done 2026-03-20T11:45:59.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cb8bc590:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cb8bc590:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:59.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 collect(): s=0x5631cbc36db0 stack=0x5631cbd79340 is complete 2026-03-20T11:45:59.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:59.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:59.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:59.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 15 stack 0x5631cbc36db0 end 2026-03-20T11:45:59.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 stack->operate() returned ret=-22 2026-03-20T11:45:59.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is done 2026-03-20T11:45:59.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbca5c20:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 15 stack 0x5631cbd72b00 end 2026-03-20T11:45:59.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 run: stack=0x5631cbd72b00 is done 2026-03-20T11:45:59.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:59.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdab220:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 collect(): s=0x5631cb8cdc90 stack=0x5631cbd72b00 is complete 2026-03-20T11:45:59.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:59.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdab220:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 15 stack 0x5631cb8cdc90 end 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 stack->operate() returned ret=-22 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 run: stack=0x5631cb8cdc90 is done 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 4 starting trim on bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdad280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdab220:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 4 starting trim on bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdad280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 enqueued request req=0x5631cbd7e210 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 req: 0x5631cbd7e210 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is io blocked 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbd76ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbd76ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 enqueued request req=0x5631cbcac160 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 req: 0x5631cbd7e210 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 req: 0x5631cbcac160 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa350c1d900 20 run: stack=0x5631cb8cdc90 is io blocked 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa305ffb640 20 dequeued request req=0x5631cbd7e210 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa305ffb640 20 RGWWQ: 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa305ffb640 20 req: 0x5631cbcac160 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa305ffb640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 : miss 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa305ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa3067fc640 20 dequeued request req=0x5631cbcac160 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa3067fc640 20 RGWWQ: empty 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa3067fc640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 : miss 2026-03-20T11:45:59.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa3067fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa3067fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa3067fc640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 info.flags=0x17 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa3067fc640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 to cache LRU end 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa3067fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa3067fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa3067fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa3067fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-1 : miss 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.272+0000 7fa3067fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa305ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa305ffb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 info.flags=0x17 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa305ffb640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 to cache LRU end 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa305ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa305ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa305ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa305ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-37 : miss 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa305ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa3067fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa3067fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-1 info.flags=0x0 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa3067fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-1 to cache LRU end 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa3067fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-1 : miss 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa3067fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa305ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa305ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-37 info.flags=0x0 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa305ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-37 to cache LRU end 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa305ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-37 : miss 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa305ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa3067fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa3067fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-1 info.flags=0x0 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa3067fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-1 to cache LRU end 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa3067fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1): adding source pipe: {s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa3067fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1): adding dest pipe: {s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa3067fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa3067fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa3067fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbd76ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbd76ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbd76ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbd76ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdab220:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbca5c20:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdab220:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 run: stack=0x5631cb8cdc90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cb8bc590:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cb8bc590:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.271 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 8165013fb7e6a422cc5e99a59a1d6022ca668113835e2c0798e633d78aeb5d5b 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:8165013fb7e6a422cc5e99a59a1d6022ca668113835e2c0798e633d78aeb5d5b 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa305ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa305ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-37 info.flags=0x0 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 8ca7e7562452f1c54a8e8f1d02ad35f57da236240cad71551bf56bdc2ce26b33 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa305ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-37 to cache LRU end 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=8ca7e7562452f1c54a8e8f1d02ad35f57da236240cad71551bf56bdc2ce26b33 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa305ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42): adding source pipe: {s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa305ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42): adding dest pipe: {s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=40, curl_handle=0x5631cbcad4d0 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa305ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa2a9ffb640 20 link_request req_data=0x5631cbcadbe0 req_data->id=40, curl_handle=0x5631cbcad4d0 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa305ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa305ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 run: stack=0x5631cbd72b00 is io blocked 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdad280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdad280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdad280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdad280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbd87c90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbdaa6f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbdaa6f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 26d56cf5c25e2d9c55219837fe572281f14b91a5c2fcdd45df7ccb712a233314 2026-03-20T11:45:59.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.273 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.273 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.273 INFO:teuthology.orchestra.run.vm02.stderr:26d56cf5c25e2d9c55219837fe572281f14b91a5c2fcdd45df7ccb712a233314 2026-03-20T11:45:59.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = fe270c3542d1d78bba3a6070d1962efea4276156be0497ca7e261a721c84b723 2026-03-20T11:45:59.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=fe270c3542d1d78bba3a6070d1962efea4276156be0497ca7e261a721c84b723 2026-03-20T11:45:59.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=41, curl_handle=0x5631cb8cf450 2026-03-20T11:45:59.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8cff10 req_data->id=41, curl_handle=0x5631cb8cf450 2026-03-20T11:45:59.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.273+0000 7fa350c1d900 20 run: stack=0x5631cbd79340 is io blocked 2026-03-20T11:45:59.273 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:59.273+0000 7fd06b324640 -1 req 16031884727644447369 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:59.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbcb5a60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbcb5a60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbd71620:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbd71620:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:59.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbcb5a60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cb8ce9e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 15 stack 0x5631cbd84c00 end 2026-03-20T11:45:59.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is done 2026-03-20T11:45:59.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbcb5a60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 collect(): s=0x5631cbd731d0 stack=0x5631cbd84c00 is complete 2026-03-20T11:45:59.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:59.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:59.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:59.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 15 stack 0x5631cbd731d0 end 2026-03-20T11:45:59.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 stack->operate() returned ret=-22 2026-03-20T11:45:59.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 run: stack=0x5631cbd731d0 is done 2026-03-20T11:45:59.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd7f210:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 15 stack 0x5631cbca2ac0 end 2026-03-20T11:45:59.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is done 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 collect(): s=0x5631cbda4490 stack=0x5631cbca2ac0 is complete 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 10 trimming bilogs for bucket=:dipepu-17[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22]) markers=[,,,,,,,,00000000005.46.18,,], shards=11 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 4 starting trim on bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbd7f210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 10 trimming bilog shard 8 of :dipepu-17[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22]) at marker 00000000005.46.18 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 run: stack=0x5631cbda4490 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbda99c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 enqueued request req=0x5631cbcad7c0 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 req: 0x5631cbcad7c0 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 run: stack=0x5631cbd731d0 is io blocked 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbda99c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22.8 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa3057fa640 20 dequeued request req=0x5631cbcad7c0 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa3057fa640 20 RGWWQ: empty 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa3057fa640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 : miss 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa3057fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is io blocked 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa3057fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa3057fa640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 info.flags=0x17 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa3057fa640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 to cache LRU end 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa3057fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa3057fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa3057fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa3057fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-7 : miss 2026-03-20T11:45:59.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.277+0000 7fa3057fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa3057fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa3057fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-7 info.flags=0x0 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa3057fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-7 to cache LRU end 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa3057fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-7 : miss 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa3057fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa3057fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa3057fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-7 info.flags=0x0 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa3057fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-7 to cache LRU end 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa3057fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12): adding source pipe: {s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa3057fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12): adding dest pipe: {s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa3057fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa3057fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa3057fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cb8ce9e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 run: stack=0x5631cbd731d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbcb5a60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbcb5a60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 0531996fdeebb069d6d6b40d2371aa207d12cac92211b22327b2be556b99c160 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:0531996fdeebb069d6d6b40d2371aa207d12cac92211b22327b2be556b99c160 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = b315d06fffdf91feae9c2edf33ad7768409bb9b41547137667699ba1c5ae4109 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b315d06fffdf91feae9c2edf33ad7768409bb9b41547137667699ba1c5ae4109 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=42, curl_handle=0x5631cb8c1990 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8c20a0 req_data->id=42, curl_handle=0x5631cb8c1990 2026-03-20T11:45:59.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is io blocked 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbda99c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbda99c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 15 stack 0x5631cbca2ac0 end 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is done 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbd7f210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 trim: trimmed bucket instance dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 15 stack 0x5631cbda4490 end 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 run: stack=0x5631cbda4490 is done 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 4 starting trim on bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 enqueued request req=0x5631cbd5d040 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 req: 0x5631cbd5d040 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa350c1d900 20 run: stack=0x5631cbda4490 is io blocked 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa304ff9640 20 dequeued request req=0x5631cbd5d040 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa304ff9640 20 RGWWQ: empty 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa304ff9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 : miss 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.278+0000 7fa304ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.277 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:59.279+0000 7fd01d288640 -1 req 64565958343697512 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.279+0000 7fa304ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.279+0000 7fa304ff9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 info.flags=0x17 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.279+0000 7fa304ff9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 to cache LRU end 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.279+0000 7fa304ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.279+0000 7fa304ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.279+0000 7fa304ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.279+0000 7fa304ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.279+0000 7fa304ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-48 : miss 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.279+0000 7fa304ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.279+0000 7fa304ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.279+0000 7fa304ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-48 info.flags=0x0 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.279+0000 7fa304ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-48 to cache LRU end 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.279+0000 7fa304ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-48 : miss 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.279+0000 7fa304ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa304ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa304ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-48 info.flags=0x0 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa304ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-48 to cache LRU end 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa304ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8): adding source pipe: {s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa304ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8): adding dest pipe: {s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa304ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa304ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa304ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd71620:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 20 run: stack=0x5631cbda4490 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cb8bffa0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cb8bffa0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 7cfcb0be7ed9419c50dca0a103f111b020e9a679fcf48c3112d18e5b2304fe4a 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:7cfcb0be7ed9419c50dca0a103f111b020e9a679fcf48c3112d18e5b2304fe4a 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 8a996c261628ab84fe89e8f987a5948286dfef830cf5de54a69f1d007abfa8d5 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=8a996c261628ab84fe89e8f987a5948286dfef830cf5de54a69f1d007abfa8d5 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=43, curl_handle=0x5631cb8bd9e0 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8be0f0 req_data->id=43, curl_handle=0x5631cb8bd9e0 2026-03-20T11:45:59.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.280+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is io blocked 2026-03-20T11:45:59.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cb8bc590:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cb8bc590:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:59.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbca5c20:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa350c1d900 15 stack 0x5631cbd72b00 end 2026-03-20T11:45:59.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa350c1d900 20 run: stack=0x5631cbd72b00 is done 2026-03-20T11:45:59.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdab220:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa350c1d900 20 collect(): s=0x5631cb8cdc90 stack=0x5631cbd72b00 is complete 2026-03-20T11:45:59.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa350c1d900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:59.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa350c1d900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:59.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdab220:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:59.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa350c1d900 15 stack 0x5631cb8cdc90 end 2026-03-20T11:45:59.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa350c1d900 20 stack->operate() returned ret=-22 2026-03-20T11:45:59.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa350c1d900 20 run: stack=0x5631cb8cdc90 is done 2026-03-20T11:45:59.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa350c1d900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:59.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdab220:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa350c1d900 4 starting trim on bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:45:59.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa350c1d900 20 enqueued request req=0x5631cbcac160 2026-03-20T11:45:59.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa350c1d900 20 req: 0x5631cbcac160 2026-03-20T11:45:59.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa350c1d900 20 run: stack=0x5631cb8cdc90 is io blocked 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa303ff7640 20 dequeued request req=0x5631cbcac160 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa303ff7640 20 RGWWQ: empty 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa303ff7640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 : miss 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa303ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa303ff7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa303ff7640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 info.flags=0x17 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa303ff7640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 to cache LRU end 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa303ff7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa303ff7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa303ff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa303ff7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-59 : miss 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.315+0000 7fa303ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbdaa6f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbdaa6f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbdaa6f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbdaa6f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbd87c90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa350c1d900 15 stack 0x5631cbd79340 end 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa350c1d900 20 run: stack=0x5631cbd79340 is done 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa350c1d900 20 collect(): s=0x5631cbc36db0 stack=0x5631cbd79340 is complete 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa350c1d900 10 trimming bilogs for bucket=:dipepu-37[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42]) markers=[,,,,,,,,,,00000000001.20.6], shards=11 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd73400:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa350c1d900 10 trimming bilog shard 10 of :dipepu-37[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42]) at marker 00000000001.20.6 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbcb6090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbcb6090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42.10 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa350c1d900 20 run: stack=0x5631cbd79340 is io blocked 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa303ff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa303ff7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-59 info.flags=0x0 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa303ff7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-59 to cache LRU end 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa303ff7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-59 : miss 2026-03-20T11:45:59.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa303ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa303ff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa303ff7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-59 info.flags=0x0 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa303ff7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-59 to cache LRU end 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa303ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19): adding source pipe: {s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa303ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19): adding dest pipe: {s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa303ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa303ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.316+0000 7fa303ff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdab220:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbdaa6f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdab220:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 20 run: stack=0x5631cb8cdc90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cb8bc590:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cb8bc590:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 3df8d2e6cceae0457fe038c458d4c0d8f5a6f996a3be9d943f2c1f5fce9db9f7 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:3df8d2e6cceae0457fe038c458d4c0d8f5a6f996a3be9d943f2c1f5fce9db9f7 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 51f5bc71a8d2aea7659b95c88639204e6019fdba3e87de4c42c940ebb091fc71 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=51f5bc71a8d2aea7659b95c88639204e6019fdba3e87de4c42c940ebb091fc71 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=44, curl_handle=0x5631cb8cf450 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8cff10 req_data->id=44, curl_handle=0x5631cb8cf450 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.317+0000 7fa350c1d900 20 run: stack=0x5631cbd72b00 is io blocked 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbcb6090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbcb6090:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa350c1d900 15 stack 0x5631cbd79340 end 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa350c1d900 20 run: stack=0x5631cbd79340 is done 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd73400:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa350c1d900 20 trim: trimmed bucket instance dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa350c1d900 15 stack 0x5631cbc36db0 end 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is done 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa350c1d900 4 starting trim on bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa350c1d900 20 enqueued request req=0x5631cbd7e210 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa350c1d900 20 req: 0x5631cbd7e210 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is io blocked 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa3047f8640 20 dequeued request req=0x5631cbd7e210 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa3047f8640 20 RGWWQ: empty 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa3047f8640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 : miss 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa3047f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa3047f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa3047f8640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 info.flags=0x17 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa3047f8640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 to cache LRU end 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa3047f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa3047f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa3047f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa3047f8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-21 : miss 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.318+0000 7fa3047f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa3047f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa3047f8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-21 info.flags=0x0 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa3047f8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-21 to cache LRU end 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa3047f8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-21 : miss 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa3047f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa3047f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa3047f8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-21 info.flags=0x0 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa3047f8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-21 to cache LRU end 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa3047f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26): adding source pipe: {s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa3047f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26): adding dest pipe: {s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa3047f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa3047f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa3047f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbd85930:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbdae080:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbdae080:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = e6e310e67b7b040729d40581f71c1b5d8e09aa83039a5a14338d01e242cbd12c 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:e6e310e67b7b040729d40581f71c1b5d8e09aa83039a5a14338d01e242cbd12c 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = ea5197226179acd5992afdf2fe14caaabc3568a6c8985547fcc2b8d4f42a99ca 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=ea5197226179acd5992afdf2fe14caaabc3568a6c8985547fcc2b8d4f42a99ca 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=45, curl_handle=0x5631cbcad4d0 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa2a9ffb640 20 link_request req_data=0x5631cbcadbe0 req_data->id=45, curl_handle=0x5631cbcad4d0 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.319+0000 7fa350c1d900 20 run: stack=0x5631cbd79340 is io blocked 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbcb5a60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbcb5a60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbcb5a60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbcb5a60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cb8ce9e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 15 stack 0x5631cbd84c00 end 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is done 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 collect(): s=0x5631cbd731d0 stack=0x5631cbd84c00 is complete 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 10 trimming bilogs for bucket=:dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) markers=[,00000000002.6.6,,00000000001.10.6,,00000000001.6.6,00000000002.6.6,00000000001.6.6,00000000001.12.6,00000000001.4.6,], shards=11 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbdaeec0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 10 trimming bilog shard 1 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000002.6.6 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 10 trimming bilog shard 3 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.10.6 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 10 trimming bilog shard 5 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.6.6 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 10 trimming bilog shard 6 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000002.6.6 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 10 trimming bilog shard 7 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.6.6 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 10 trimming bilog shard 8 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.12.6 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 10 trimming bilog shard 9 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.4.6 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 run: stack=0x5631cbd731d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda1d00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 cr:s=0x5631cbd882d0:op=0x5631cbca4740:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbda0c60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cbda68f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 cr:s=0x5631cbca6250:op=0x5631cbd73d10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 cr:s=0x5631cbd86870:op=0x5631cbda8030:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 cr:s=0x5631cbdae6b0:op=0x5631cbdafef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda1d00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.1 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is io blocked 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 cr:s=0x5631cbd882d0:op=0x5631cbca4740:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.3 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 run: stack=0x5631cbd882d0 is io blocked 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbda0c60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.5 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 run: stack=0x5631cbca8810 is io blocked 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cbda68f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.6 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 run: stack=0x5631cbca9120 is io blocked 2026-03-20T11:45:59.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 cr:s=0x5631cbca6250:op=0x5631cbd73d10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.7 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 run: stack=0x5631cbca6250 is io blocked 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 cr:s=0x5631cbd86870:op=0x5631cbda8030:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.8 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 run: stack=0x5631cbd86870 is io blocked 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 cr:s=0x5631cbdae6b0:op=0x5631cbdafef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.9 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.320+0000 7fa350c1d900 20 run: stack=0x5631cbdae6b0 is io blocked 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.321+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cb8bffa0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.321+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cb8bffa0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.321+0000 7fa350c1d900 20 cr:s=0x5631cbca2ac0:op=0x5631cbd71620:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.321+0000 7fa350c1d900 15 stack 0x5631cbca2ac0 end 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.321+0000 7fa350c1d900 20 run: stack=0x5631cbca2ac0 is done 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.321+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.321+0000 7fa350c1d900 20 collect(): s=0x5631cbda4490 stack=0x5631cbca2ac0 is complete 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.321+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.321+0000 7fa350c1d900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.321+0000 7fa350c1d900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.321+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.321+0000 7fa350c1d900 15 stack 0x5631cbda4490 end 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.321+0000 7fa350c1d900 20 stack->operate() returned ret=-22 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.321+0000 7fa350c1d900 20 run: stack=0x5631cbda4490 is done 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.321+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.321+0000 7fa350c1d900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.321+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.321+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.321+0000 7fa350c1d900 4 starting trim on bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.321+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.321+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.321+0000 7fa350c1d900 20 enqueued request req=0x5631cbd5d040 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.321+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.321+0000 7fa350c1d900 20 req: 0x5631cbd5d040 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.321+0000 7fa350c1d900 20 run: stack=0x5631cbda4490 is io blocked 2026-03-20T11:45:59.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cbda68f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cbda68f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 15 stack 0x5631cbca9120 end 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 run: stack=0x5631cbca9120 is done 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbdaeec0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 run: stack=0x5631cbd731d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda1d00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda1d00:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 15 stack 0x5631cbd84c00 end 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is done 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbdaeec0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 run: stack=0x5631cbd731d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa3037f6640 20 dequeued request req=0x5631cbd5d040 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa3037f6640 20 RGWWQ: empty 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa3037f6640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 : miss 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa3037f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 cr:s=0x5631cbca6250:op=0x5631cbd73d10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 cr:s=0x5631cbca6250:op=0x5631cbd73d10:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 15 stack 0x5631cbca6250 end 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 run: stack=0x5631cbca6250 is done 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbdaeec0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 run: stack=0x5631cbd731d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 cr:s=0x5631cbd882d0:op=0x5631cbca4740:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 cr:s=0x5631cbd882d0:op=0x5631cbca4740:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 15 stack 0x5631cbd882d0 end 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 run: stack=0x5631cbd882d0 is done 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbdaeec0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 run: stack=0x5631cbd731d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 cr:s=0x5631cbd86870:op=0x5631cbda8030:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 cr:s=0x5631cbd86870:op=0x5631cbda8030:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 15 stack 0x5631cbd86870 end 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 run: stack=0x5631cbd86870 is done 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbdaeec0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 run: stack=0x5631cbd731d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa3037f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa3037f6640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 info.flags=0x17 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa3037f6640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 to cache LRU end 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa3037f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa3037f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa3037f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa3037f6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-41 : miss 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa3037f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbda0c60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbda0c60:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 15 stack 0x5631cbca8810 end 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 run: stack=0x5631cbca8810 is done 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbdaeec0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 run: stack=0x5631cbd731d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa3037f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa3037f6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-41 info.flags=0x0 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa3037f6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-41 to cache LRU end 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa3037f6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-41 : miss 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa3037f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 cr:s=0x5631cbdae6b0:op=0x5631cbdafef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 cr:s=0x5631cbdae6b0:op=0x5631cbdafef0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 15 stack 0x5631cbdae6b0 end 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 run: stack=0x5631cbdae6b0 is done 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbdaeec0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 cr:s=0x5631cbd731d0:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 trim: trimmed bucket instance dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 15 stack 0x5631cbd731d0 end 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 run: stack=0x5631cbd731d0 is done 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 4 starting trim on bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd7c0e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd7c0e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 enqueued request req=0x5631cb8cce00 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 req: 0x5631cb8cce00 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa350c1d900 20 run: stack=0x5631cbca8810 is io blocked 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa302ff5640 20 dequeued request req=0x5631cb8cce00 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa302ff5640 20 RGWWQ: empty 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa302ff5640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 : miss 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.322+0000 7fa302ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa3037f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa3037f6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-41 info.flags=0x0 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa3037f6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-41 to cache LRU end 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa3037f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1): adding source pipe: {s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa3037f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1): adding dest pipe: {s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa3037f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa3037f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa3037f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbda7860:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 cr:s=0x5631cbd86870:op=0x5631cbd71620:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 run: stack=0x5631cbda4490 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 cr:s=0x5631cbd86870:op=0x5631cb8bffa0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 cr:s=0x5631cbd86870:op=0x5631cb8bffa0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa302ff5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 884364a58d65cbd7f4d6c92136d2c82e9cf8ca8276af8d622f43a16ccec72278 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa302ff5640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 info.flags=0x17 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa302ff5640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 to cache LRU end 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa302ff5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa302ff5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:884364a58d65cbd7f4d6c92136d2c82e9cf8ca8276af8d622f43a16ccec72278 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa302ff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa302ff5640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-49 : miss 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa302ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = b151b3fe797631bd61f66e2eb65fdb1f51d4e8a003fe4d2345ed1b1868c80574 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b151b3fe797631bd61f66e2eb65fdb1f51d4e8a003fe4d2345ed1b1868c80574 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=46, curl_handle=0x5631cb8bd9e0 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8be0f0 req_data->id=46, curl_handle=0x5631cb8bd9e0 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 run: stack=0x5631cbd86870 is io blocked 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa302ff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa302ff5640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-49 info.flags=0x0 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa302ff5640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-49 to cache LRU end 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa302ff5640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-49 : miss 2026-03-20T11:45:59.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa302ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.322 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:59.322+0000 7fd02aaa3640 -1 req 2721668692060919251 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa302ff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa302ff5640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-49 info.flags=0x0 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa302ff5640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-49 to cache LRU end 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa302ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9): adding source pipe: {s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa302ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9): adding dest pipe: {s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa302ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa302ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa302ff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd7c0e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd7c0e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd7c0e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd7c0e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 cr:s=0x5631cbd882d0:op=0x5631cbd76490:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 run: stack=0x5631cbca8810 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 cr:s=0x5631cbd882d0:op=0x5631cbd87180:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 cr:s=0x5631cbd882d0:op=0x5631cbd87180:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 6102c37fce759971b87666620e5f9e6689e623e7c69e52b3084b338e8c3520e7 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:6102c37fce759971b87666620e5f9e6689e623e7c69e52b3084b338e8c3520e7 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 880b6bef7657d557784bcffa4700d56af16a48f98dacedb8f5e3cc85aea4356d 2026-03-20T11:45:59.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=880b6bef7657d557784bcffa4700d56af16a48f98dacedb8f5e3cc85aea4356d 2026-03-20T11:45:59.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=47, curl_handle=0x5631cb8c1990 2026-03-20T11:45:59.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8c20a0 req_data->id=47, curl_handle=0x5631cb8c1990 2026-03-20T11:45:59.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.323+0000 7fa350c1d900 20 run: stack=0x5631cbd882d0 is io blocked 2026-03-20T11:45:59.356 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cb8bc590:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.356 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cb8bc590:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.356 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cb8bc590:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.356 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cb8bc590:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.356 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbdaa6f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.356 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 15 stack 0x5631cbd72b00 end 2026-03-20T11:45:59.356 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 20 run: stack=0x5631cbd72b00 is done 2026-03-20T11:45:59.356 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdab220:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.356 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 20 collect(): s=0x5631cb8cdc90 stack=0x5631cbd72b00 is complete 2026-03-20T11:45:59.356 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.356 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 10 trimming bilogs for bucket=:dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) markers=[,00000000001.44.6,,,,00000000001.24.6,,,00000000001.44.6,,00000000001.39.6], shards=11 2026-03-20T11:45:59.356 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdaa6f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.356 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 10 trimming bilog shard 1 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.44.6 2026-03-20T11:45:59.356 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 10 trimming bilog shard 5 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.24.6 2026-03-20T11:45:59.356 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 10 trimming bilog shard 8 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.44.6 2026-03-20T11:45:59.356 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 10 trimming bilog shard 10 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.39.6 2026-03-20T11:45:59.356 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 20 run: stack=0x5631cb8cdc90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.356 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbca4740:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.356 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 20 cr:s=0x5631cbca6250:op=0x5631cbdaf0a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.356 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda8030:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.356 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cbd877b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.356 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbca4740:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.356 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.1 2026-03-20T11:45:59.357 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 20 run: stack=0x5631cbd72b00 is io blocked 2026-03-20T11:45:59.357 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 20 cr:s=0x5631cbca6250:op=0x5631cbdaf0a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.357 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.5 2026-03-20T11:45:59.357 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 20 run: stack=0x5631cbca6250 is io blocked 2026-03-20T11:45:59.357 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda8030:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.357 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.8 2026-03-20T11:45:59.357 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is io blocked 2026-03-20T11:45:59.357 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cbd877b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.357 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.10 2026-03-20T11:45:59.357 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.359+0000 7fa350c1d900 20 run: stack=0x5631cbca9120 is io blocked 2026-03-20T11:45:59.357 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.360+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbca4740:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.357 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.360+0000 7fa350c1d900 20 cr:s=0x5631cbd72b00:op=0x5631cbca4740:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.357 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.360+0000 7fa350c1d900 15 stack 0x5631cbd72b00 end 2026-03-20T11:45:59.357 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.360+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.357 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.360+0000 7fa350c1d900 20 run: stack=0x5631cbd72b00 is done 2026-03-20T11:45:59.357 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.360+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdaa6f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.357 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.360+0000 7fa350c1d900 20 run: stack=0x5631cb8cdc90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.357 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.360+0000 7fa350c1d900 20 cr:s=0x5631cbca6250:op=0x5631cbdaf0a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.357 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.360+0000 7fa350c1d900 20 cr:s=0x5631cbca6250:op=0x5631cbdaf0a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.357 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.360+0000 7fa350c1d900 15 stack 0x5631cbca6250 end 2026-03-20T11:45:59.357 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.360+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.357 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.360+0000 7fa350c1d900 20 run: stack=0x5631cbca6250 is done 2026-03-20T11:45:59.357 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.360+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdaa6f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.357 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.360+0000 7fa350c1d900 20 run: stack=0x5631cb8cdc90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cbd877b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa350c1d900 20 cr:s=0x5631cbca9120:op=0x5631cbd877b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa350c1d900 15 stack 0x5631cbca9120 end 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa350c1d900 20 run: stack=0x5631cbca9120 is done 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdaa6f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa350c1d900 20 run: stack=0x5631cb8cdc90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda8030:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda8030:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa350c1d900 15 stack 0x5631cbd84c00 end 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is done 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdaa6f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdab220:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa350c1d900 20 trim: trimmed bucket instance dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa350c1d900 15 stack 0x5631cb8cdc90 end 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa350c1d900 20 run: stack=0x5631cb8cdc90 is done 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdab220:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa350c1d900 4 starting trim on bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbda1a10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbda1a10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa350c1d900 20 enqueued request req=0x5631cbd73800 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa350c1d900 20 req: 0x5631cbd73800 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa350c1d900 20 run: stack=0x5631cb8cdc90 is io blocked 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa3027f4640 20 dequeued request req=0x5631cbd73800 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa3027f4640 20 RGWWQ: empty 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa3027f4640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 : miss 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa3027f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa3027f4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa3027f4640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 info.flags=0x17 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa3027f4640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 to cache LRU end 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa3027f4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa3027f4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa3027f4640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa3027f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa3027f4640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-45 : miss 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa3027f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa3027f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa3027f4640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-45 info.flags=0x0 2026-03-20T11:45:59.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa3027f4640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-45 to cache LRU end 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa3027f4640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-45 : miss 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.361+0000 7fa3027f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa3027f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbdae080:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa3027f4640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-45 info.flags=0x0 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa3027f4640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-45 to cache LRU end 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa3027f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5): adding source pipe: {s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa3027f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5): adding dest pipe: {s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa3027f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa3027f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa3027f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbdae080:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbda1a10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbdae080:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbda1a10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbdae080:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbda1a10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbd85930:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 15 stack 0x5631cbd79340 end 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 run: stack=0x5631cbd79340 is done 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbda1a10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 collect(): s=0x5631cbc36db0 stack=0x5631cbd79340 is complete 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 10 trimming bilogs for bucket=:dipepu-21[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26]) markers=[,,,,,,,,00000000001.60.14,,], shards=11 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdab220:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cb8bc590:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 10 trimming bilog shard 8 of :dipepu-21[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26]) at marker 00000000001.60.14 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbdaa6f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdab220:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 run: stack=0x5631cb8cdc90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbca4740:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbdae080:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbca4740:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26.8 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is io blocked 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbdae080:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 8499e2489de3590ffd6b7c2f909be43f1bf73156bf12bacfa15ebfce325f0112 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:8499e2489de3590ffd6b7c2f909be43f1bf73156bf12bacfa15ebfce325f0112 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 0053687c33481a6ac54000eb88e9714d5c05c3d0dbde16004751c16afce51a45 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=0053687c33481a6ac54000eb88e9714d5c05c3d0dbde16004751c16afce51a45 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=48, curl_handle=0x5631cbcad4d0 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa2a9ffb640 20 link_request req_data=0x5631cbcadbe0 req_data->id=48, curl_handle=0x5631cbcad4d0 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.362+0000 7fa350c1d900 20 run: stack=0x5631cbd79340 is io blocked 2026-03-20T11:45:59.360 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:59.362+0000 7fd02e2aa640 -1 req 17412216102512601864 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbca4740:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbca4740:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa350c1d900 15 stack 0x5631cbd84c00 end 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa350c1d900 20 stack->operate() returned ret=-61 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is done 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cb8bc590:24BucketTrimShardCollectCR: operate() 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa350c1d900 20 trim: trimmed bucket instance dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa350c1d900 15 stack 0x5631cbc36db0 end 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is done 2026-03-20T11:45:59.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa350c1d900 4 starting trim on bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdad280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdad280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa350c1d900 20 enqueued request req=0x5631cbd7e210 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa350c1d900 20 RGWWQ: 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa350c1d900 20 req: 0x5631cbd7e210 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is io blocked 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa301ff3640 20 dequeued request req=0x5631cbd7e210 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa301ff3640 20 RGWWQ: empty 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa301ff3640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 : miss 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa301ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa301ff3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa301ff3640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 info.flags=0x17 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa301ff3640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 to cache LRU end 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa301ff3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa301ff3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa301ff3640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa301ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa301ff3640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-33 : miss 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.363+0000 7fa301ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa301ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa301ff3640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-33 info.flags=0x0 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa301ff3640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-33 to cache LRU end 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa301ff3640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-33 : miss 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa301ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa301ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa301ff3640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-33 info.flags=0x0 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa301ff3640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-33 to cache LRU end 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa301ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38): adding source pipe: {s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa301ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38): adding dest pipe: {s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa301ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa301ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa301ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdad280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdad280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdad280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbdad280:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda9ba0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cb8bc590:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cb8bc590:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.361 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:45:59 +0000 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114559Z 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = dfdaa24dd080f33c4c2c9e200b68bcbde4baa7e98efe576f4be055aede9633a6 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114559Z 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:dfdaa24dd080f33c4c2c9e200b68bcbde4baa7e98efe576f4be055aede9633a6 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 394377a6e282ae4f971b1042f645848fda44f527438f1b3ffb72f45bdb7bf040 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=394377a6e282ae4f971b1042f645848fda44f527438f1b3ffb72f45bdb7bf040 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114559Z 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 20 register_request mgr=0x7ffde2067710 req_data->id=49, curl_handle=0x5631cb8cf450 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa2a9ffb640 20 link_request req_data=0x5631cb8cff10 req_data->id=49, curl_handle=0x5631cb8cf450 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.364+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is io blocked 2026-03-20T11:45:59.362 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:45:59.363+0000 7fd01e28a640 -1 req 14502602616119548213 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 20 cr:s=0x5631cbd86870:op=0x5631cb8bffa0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 20 cr:s=0x5631cbd86870:op=0x5631cb8bffa0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 20 cr:s=0x5631cbd86870:op=0x5631cbd71620:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 15 stack 0x5631cbd86870 end 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 20 run: stack=0x5631cbd86870 is done 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 20 cr:s=0x5631cbd882d0:op=0x5631cbd87180:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 20 collect(): s=0x5631cbda4490 stack=0x5631cbd86870 is complete 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 20 cr:s=0x5631cbda4490:op=0x5631cbca1340:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 15 stack 0x5631cbda4490 end 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 20 stack->operate() returned ret=-22 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 20 run: stack=0x5631cbda4490 is done 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 20 cr:s=0x5631cbd882d0:op=0x5631cbd87180:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 20 cr:s=0x5631cbd882d0:op=0x5631cbd87180:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 20 cr:s=0x5631cbd882d0:op=0x5631cbd87180:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 20 cr:s=0x5631cbd882d0:op=0x5631cbd76490:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 15 stack 0x5631cbd882d0 end 2026-03-20T11:45:59.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 20 run: stack=0x5631cbd882d0 is done 2026-03-20T11:45:59.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 20 collect(): s=0x5631cbca8810 stack=0x5631cbd882d0 is complete 2026-03-20T11:45:59.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:59.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:59.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 20 cr:s=0x5631cbca8810:op=0x5631cbd837a0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:59.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 15 stack 0x5631cbca8810 end 2026-03-20T11:45:59.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 20 stack->operate() returned ret=-22 2026-03-20T11:45:59.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 20 run: stack=0x5631cbca8810 is done 2026-03-20T11:45:59.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:59.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.365+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbdae080:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbdae080:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 20 cr:s=0x5631cbd79340:op=0x5631cbdaa6f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 15 stack 0x5631cbd79340 end 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 20 run: stack=0x5631cbd79340 is done 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cb8bc590:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cb8bc590:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdab220:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 20 collect(): s=0x5631cb8cdc90 stack=0x5631cbd79340 is complete 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 20 cr:s=0x5631cb8cdc90:op=0x5631cbdab220:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 15 stack 0x5631cb8cdc90 end 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 20 stack->operate() returned ret=-22 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 20 run: stack=0x5631cb8cdc90 is done 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 20 cr:s=0x5631cbd84c00:op=0x5631cbda9ba0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 15 stack 0x5631cbd84c00 end 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 20 run: stack=0x5631cbd84c00 is done 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 20 collect(): s=0x5631cbc36db0 stack=0x5631cbd84c00 is complete 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 10 min_generation is 0 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 4 failed to correlate bucket sync status from peers 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 20 cr:s=0x5631cbc36db0:op=0x5631cbd77ca0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 15 stack 0x5631cbc36db0 end 2026-03-20T11:45:59.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 20 stack->operate() returned ret=-22 2026-03-20T11:45:59.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 20 run: stack=0x5631cbc36db0 is done 2026-03-20T11:45:59.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:45:59.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:45:59.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbca2e40:27BucketTrimInstanceCollectCR: operate() returned r=-22 2026-03-20T11:45:59.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbc9f5e0:12BucketTrimCR: operate() 2026-03-20T11:45:59.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 20 writing bucket trim marker=12:9c77f2bb:root::dipepu-31:head 2026-03-20T11:45:59.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbd76490:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:59.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbd76490:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:59.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.405+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is io blocked 2026-03-20T11:45:59.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.407+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbd76490:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:59.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.407+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbd76490:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:59.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.407+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbd76490:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:59.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.407+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbd76490:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:45:59.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.407+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbc9f5e0:12BucketTrimCR: operate() 2026-03-20T11:45:59.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.407+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbd76490:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:59.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.407+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbd76490:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:59.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.407+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is io blocked 2026-03-20T11:45:59.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.407+0000 7fa322ffd640 20 trim: bucket trim completed 2026-03-20T11:45:59.405 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.408+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbd76490:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:59.405 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.408+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbd76490:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:59.405 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.408+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbd76490:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:59.405 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.408+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbd76490:16RGWRadosNotifyCR: operate() 2026-03-20T11:45:59.405 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.408+0000 7fa350c1d900 20 cr:s=0x5631cbd37d40:op=0x5631cbc9f5e0:12BucketTrimCR: operate() 2026-03-20T11:45:59.405 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.408+0000 7fa350c1d900 4 bucket index log processing completed in 0.714968681s 2026-03-20T11:45:59.405 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.408+0000 7fa350c1d900 15 stack 0x5631cbd37d40 end 2026-03-20T11:45:59.405 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.408+0000 7fa350c1d900 20 run: stack=0x5631cbd37d40 is done 2026-03-20T11:45:59.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.413+0000 7fa350c1d900 20 remove_watcher() i=1 2026-03-20T11:45:59.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.413+0000 7fa350c1d900 2 removed watcher, disabling cache 2026-03-20T11:45:59.412 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.415+0000 7fa350c1d900 20 remove_watcher() i=2 2026-03-20T11:45:59.412 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.415+0000 7fa350c1d900 20 remove_watcher() i=0 2026-03-20T11:45:59.412 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.415+0000 7fa350c1d900 20 remove_watcher() i=4 2026-03-20T11:45:59.412 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.415+0000 7fa350c1d900 20 remove_watcher() i=3 2026-03-20T11:45:59.412 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.415+0000 7fa350c1d900 20 remove_watcher() i=7 2026-03-20T11:45:59.412 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.415+0000 7fa350c1d900 20 remove_watcher() i=5 2026-03-20T11:45:59.412 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:45:59.415+0000 7fa350c1d900 20 remove_watcher() i=6 2026-03-20T11:46:04.418 INFO:rgw_multi.tests:trimming on zone=a2 2026-03-20T11:46:04.418 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '50', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:46:04.418 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '50', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:46:04.418 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bilog autotrim --rgw-sync-log-trim-max-buckets 50 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 20 --debug-ms 0 2026-03-20T11:46:04.454 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:04.454 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:04.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.473+0000 7fb9cb610900 20 rados->read ofs=0 len=0 2026-03-20T11:46:04.471 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.474+0000 7fb9cb610900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:46:04.471 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.474+0000 7fb9cb610900 20 rados->read ofs=0 len=0 2026-03-20T11:46:04.471 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.474+0000 7fb9cb610900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:46:04.471 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.474+0000 7fb9cb610900 20 realm test-realm 6a789b89-c2b8-45f5-82cc-fb80234d0c6c 2026-03-20T11:46:04.471 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.474+0000 7fb9cb610900 20 rados->read ofs=0 len=0 2026-03-20T11:46:04.471 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.475+0000 7fb9cb610900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:46:04.472 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.475+0000 7fb9cb610900 20 rados->read ofs=0 len=0 2026-03-20T11:46:04.472 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.476+0000 7fb9cb610900 20 rados_obj.operate() r=0 bl.length=10 2026-03-20T11:46:04.472 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.476+0000 7fb9cb610900 20 rados->read ofs=0 len=0 2026-03-20T11:46:04.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.476+0000 7fb9cb610900 20 rados_obj.operate() r=0 bl.length=2020 2026-03-20T11:46:04.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.476+0000 7fb9cb610900 20 rados->read ofs=0 len=0 2026-03-20T11:46:04.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.476+0000 7fb9cb610900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:46:04.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.476+0000 7fb9cb610900 20 rados->read ofs=0 len=0 2026-03-20T11:46:04.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.477+0000 7fb9cb610900 20 rados_obj.operate() r=0 bl.length=1030 2026-03-20T11:46:04.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.477+0000 7fb9cb610900 20 zone a2 found 2026-03-20T11:46:04.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.477+0000 7fb9cb610900 4 Realm: test-realm (6a789b89-c2b8-45f5-82cc-fb80234d0c6c) 2026-03-20T11:46:04.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.477+0000 7fb9cb610900 4 ZoneGroup: a (a) 2026-03-20T11:46:04.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.477+0000 7fb9cb610900 4 Zone: a2 (51055576-6662-409d-8abb-ecaeefd3dd80) 2026-03-20T11:46:04.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.477+0000 7fb9cb610900 4 using period configuration: cb2fcdce-2c1b-429a-8601-a284ee3a2381:12 2026-03-20T11:46:04.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.477+0000 7fb9cb610900 20 period zonegroup name a 2026-03-20T11:46:04.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.477+0000 7fb9cb610900 20 using current period zonegroup a 2026-03-20T11:46:04.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.477+0000 7fb9cb610900 20 rados->read ofs=0 len=0 2026-03-20T11:46:04.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.477+0000 7fb9cb610900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.477+0000 7fb9cb610900 20 rados->read ofs=0 len=0 2026-03-20T11:46:04.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.478+0000 7fb9cb610900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.478+0000 7fb9cb610900 20 init(): pipe_map (zone=4b634ebf-58f5-45bc-839a-cb4b425e773a bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:46:04.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.478+0000 7fb9cb610900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.478+0000 7fb9cb610900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.478+0000 7fb9cb610900 20 rados->read ofs=0 len=0 2026-03-20T11:46:04.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.478+0000 7fb9cb610900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.478+0000 7fb9cb610900 20 rados->read ofs=0 len=0 2026-03-20T11:46:04.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.478+0000 7fb9cb610900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.478+0000 7fb9cb610900 20 init(): pipe_map (zone=51055576-6662-409d-8abb-ecaeefd3dd80 bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:46:04.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.478+0000 7fb9cb610900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.478+0000 7fb9cb610900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.478+0000 7fb9cb610900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.478+0000 7fb9cb610900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.478+0000 7fb9cb610900 20 started sync module instance, tier type = 2026-03-20T11:46:04.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.478+0000 7fb9cb610900 20 generating connection object for zone a1 id 4b634ebf-58f5-45bc-839a-cb4b425e773a 2026-03-20T11:46:04.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.478+0000 7fb9cb610900 20 started zone id=51055576-6662-409d-8abb-ecaeefd3dd80 (name=a2) with tier type = 2026-03-20T11:46:04.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.482+0000 7fb9cb610900 20 add_watcher() i=0 2026-03-20T11:46:04.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.483+0000 7fb9cb610900 20 add_watcher() i=2 2026-03-20T11:46:04.480 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.483+0000 7fb9cb610900 20 add_watcher() i=6 2026-03-20T11:46:04.480 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.483+0000 7fb9cb610900 20 add_watcher() i=3 2026-03-20T11:46:04.480 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.483+0000 7fb9cb610900 20 add_watcher() i=7 2026-03-20T11:46:04.481 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.484+0000 7fb9cb610900 20 add_watcher() i=1 2026-03-20T11:46:04.481 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.484+0000 7fb9cb610900 20 add_watcher() i=4 2026-03-20T11:46:04.481 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.485+0000 7fb9cb610900 20 add_watcher() i=5 2026-03-20T11:46:04.481 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.485+0000 7fb9cb610900 2 all 8 watchers are set, enabling cache 2026-03-20T11:46:04.483 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.487+0000 7fb9cb610900 20 rgw_check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2026-03-20T11:46:04.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.487+0000 7fb9cb610900 20 rgw_check_secure_mon_conn(): mode 1 is insecure 2026-03-20T11:46:04.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.487+0000 7fb9cb610900 5 note: GC not initialized 2026-03-20T11:46:04.537 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.540+0000 7fb9cb610900 20 init_complete bucket index max shards: 11 2026-03-20T11:46:04.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.540+0000 7fb9cb610900 20 Filter name: none 2026-03-20T11:46:04.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.541+0000 7fb974fe1640 20 reqs_thread_entry: start 2026-03-20T11:46:04.538 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.541+0000 7fb926ffd640 20 reqs_thread_entry: start 2026-03-20T11:46:04.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb9cb610900 10 rgw bucket trim manager: Watching bilog.trim 2026-03-20T11:46:04.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f270290:12BucketTrimCR: operate() 2026-03-20T11:46:04.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb9cb610900 10 fetching active bucket counters 2026-03-20T11:46:04.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f272dc0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:04.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f272dc0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:04.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is io blocked 2026-03-20T11:46:04.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb9a27fc640 20 trim: get_bucket_counters: [] 2026-03-20T11:46:04.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f272dc0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:04.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f272dc0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:04.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f272dc0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:04.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f272dc0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:04.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f270290:12BucketTrimCR: operate() 2026-03-20T11:46:04.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:04.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:04.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is io blocked 2026-03-20T11:46:04.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:04.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:04.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:04.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:04.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f270290:12BucketTrimCR: operate() 2026-03-20T11:46:04.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb9cb610900 10 listing cold buckets from marker= 2026-03-20T11:46:04.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:14MetadataListCR: operate() 2026-03-20T11:46:04.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:14MetadataListCR: operate() 2026-03-20T11:46:04.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb9cb610900 20 enqueued request req=0x7fb9840029e0 2026-03-20T11:46:04.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb9cb610900 20 req: 0x7fb9840029e0 2026-03-20T11:46:04.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is io blocked 2026-03-20T11:46:04.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb98effd640 20 dequeued request req=0x7fb9840029e0 2026-03-20T11:46:04.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.543+0000 7fb98effd640 20 RGWWQ: empty 2026-03-20T11:46:04.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.544+0000 7fb98effd640 20 rgw async rados processor: starting metadata listing at 2026-03-20T11:46:04.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.551+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-10 2026-03-20T11:46:04.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.551+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:04.554 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.557+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:04.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.563+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:04.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.569+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:04.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.575+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:04.578 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.581+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:04.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.587+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-19 2026-03-20T11:46:04.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.587+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-3 2026-03-20T11:46:04.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.587+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:04.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.593+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:04.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.598+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-17 2026-03-20T11:46:04.595 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.598+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:04.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.604+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-38 2026-03-20T11:46:04.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.604+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:04.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.609+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-24 2026-03-20T11:46:04.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.609+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-36 2026-03-20T11:46:04.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.609+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:04.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.614+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:04.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.620+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:04.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.625+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-44 2026-03-20T11:46:04.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.625+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-35 2026-03-20T11:46:04.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.625+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:04.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.630+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-16 2026-03-20T11:46:04.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.630+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:04.632 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.635+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:04.637 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.640+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 2026-03-20T11:46:04.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.645+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:04.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.650+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-55 2026-03-20T11:46:04.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.650+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-13 2026-03-20T11:46:04.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.650+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-48 2026-03-20T11:46:04.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.650+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-25 2026-03-20T11:46:04.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.650+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-51 2026-03-20T11:46:04.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.650+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:04.651 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.654+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:04.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.658+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-46 2026-03-20T11:46:04.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.658+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:46:04.659 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.662+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:46:04.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.666+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:46:04.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.670+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-26 2026-03-20T11:46:04.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.670+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-20 2026-03-20T11:46:04.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.670+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:46:04.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.674+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:46:04.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.678+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-1 2026-03-20T11:46:04.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.678+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:46:04.678 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.681+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:46:04.682 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.685+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:46:04.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.688+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-52 2026-03-20T11:46:04.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.688+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-32 2026-03-20T11:46:04.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.688+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-42 2026-03-20T11:46:04.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.688+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:46:04.688 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.691+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:46:04.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.695+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-7 2026-03-20T11:46:04.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.695+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-54 2026-03-20T11:46:04.691 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.695+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-49 2026-03-20T11:46:04.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.695+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-47 2026-03-20T11:46:04.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.695+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 2026-03-20T11:46:04.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.698+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:46:04.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.700+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-11 2026-03-20T11:46:04.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.700+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-12 2026-03-20T11:46:04.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.700+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-41 2026-03-20T11:46:04.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.700+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-23 2026-03-20T11:46:04.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.700+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-15 2026-03-20T11:46:04.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.700+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-45 2026-03-20T11:46:04.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.700+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:46:04.700 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.703+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:46:04.702 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.705+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-9 2026-03-20T11:46:04.702 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.705+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:46:04.705 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.708+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-29 2026-03-20T11:46:04.705 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.708+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:46:04.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.710+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-31 2026-03-20T11:46:04.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.710+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:46:04.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.712+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-33 2026-03-20T11:46:04.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.712+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:04.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.714+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-4 2026-03-20T11:46:04.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.714+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:46:04.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.716+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:04.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.718+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:04.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.721+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-43 2026-03-20T11:46:04.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.721+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-30 2026-03-20T11:46:04.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.721+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-18 2026-03-20T11:46:04.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.721+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-2 2026-03-20T11:46:04.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.721+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:46:04.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.722+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-8 2026-03-20T11:46:04.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.722+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:46:04.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.724+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:04.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.726+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-58 2026-03-20T11:46:04.722 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.726+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:46:04.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.727+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb98effd640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:14MetadataListCR: operate() 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:14MetadataListCR: operate() 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:14MetadataListCR: operate() 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:14MetadataListCR: operate() 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f270290:12BucketTrimCR: operate() 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 4 collected 50 buckets for trim 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 cr:s=0x55995f294be0:op=0x55995f271db0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 4 starting trim on bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f275fe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 4 starting trim on bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2772b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 4 starting trim on bucket=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f278580:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 4 starting trim on bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 cr:s=0x55995f294be0:op=0x55995f279850:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f279fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f27a6f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f27b110:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 cr:s=0x55995f294be0:op=0x55995f279850:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 enqueued request req=0x55995f27bac0 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 req: 0x55995f27bac0 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 run: stack=0x55995f294be0 is io blocked 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f279fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 enqueued request req=0x55995f27be60 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 req: 0x55995f27bac0 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 req: 0x55995f27be60 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is io blocked 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f27a6f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 enqueued request req=0x55995f27c280 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 req: 0x55995f27bac0 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 req: 0x55995f27be60 2026-03-20T11:46:04.726 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 req: 0x55995f27c280 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 run: stack=0x55995f274bf0 is io blocked 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f27b110:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 enqueued request req=0x55995f27c6a0 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 req: 0x55995f27bac0 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 req: 0x55995f27be60 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 req: 0x55995f27c280 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 req: 0x55995f27c6a0 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb9cb610900 20 run: stack=0x55995f273080 is io blocked 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb98d7fa640 20 dequeued request req=0x55995f27bac0 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb98d7fa640 20 RGWWQ: 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb98d7fa640 20 req: 0x55995f27be60 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb98d7fa640 20 req: 0x55995f27c280 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb98d7fa640 20 req: 0x55995f27c6a0 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb98d7fa640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 : miss 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.728+0000 7fb98d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98cff9640 20 dequeued request req=0x55995f27be60 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98cff9640 20 RGWWQ: 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98cff9640 20 req: 0x55995f27c280 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98cff9640 20 req: 0x55995f27c6a0 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98cff9640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 : miss 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98dffb640 20 dequeued request req=0x55995f27c280 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98dffb640 20 RGWWQ: 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98dffb640 20 req: 0x55995f27c6a0 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98dffb640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 : miss 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98e7fc640 20 dequeued request req=0x55995f27c6a0 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98e7fc640 20 RGWWQ: empty 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98e7fc640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 : miss 2026-03-20T11:46:04.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98d7fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98d7fa640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 info.flags=0x17 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98d7fa640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 to cache LRU end 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98d7fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98d7fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98d7fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98cff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98dffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=387 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98d7fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-54 : miss 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98cff9640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 info.flags=0x17 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98cff9640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 to cache LRU end 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98cff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98cff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98cff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98cff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-53 : miss 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98dffb640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 info.flags=0x17 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98dffb640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 to cache LRU end 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98dffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98dffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98dffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98dffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-60 : miss 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98e7fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98e7fc640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 info.flags=0x17 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98e7fc640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 to cache LRU end 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98e7fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98e7fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98e7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98e7fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-50 : miss 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98cff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98cff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-53 info.flags=0x0 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98cff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-53 to cache LRU end 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98cff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-53 : miss 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.729+0000 7fb98cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98dffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98dffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-60 info.flags=0x0 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98dffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-60 to cache LRU end 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98dffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-60 : miss 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98e7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98e7fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-50 info.flags=0x0 2026-03-20T11:46:04.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98e7fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-50 to cache LRU end 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98e7fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-50 : miss 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98d7fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98d7fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-54 info.flags=0x0 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98d7fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-54 to cache LRU end 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98d7fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-54 : miss 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98d7fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98d7fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-54 info.flags=0x0 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98d7fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-54 to cache LRU end 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14): adding source pipe: {s={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14): adding dest pipe: {s={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98d7fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98dffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98dffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-60 info.flags=0x0 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f294be0:op=0x55995f279850:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98dffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-60 to cache LRU end 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f294be0:op=0x55995f279850:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f294be0:op=0x55995f279850:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f294be0:op=0x55995f279850:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20): adding source pipe: {s={b=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f294be0:op=0x55995f271db0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20): adding dest pipe: {s={b=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f275e40:op=0x55995f27cac0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f294be0:op=0x55995f271db0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 run: stack=0x55995f294be0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f275e40:op=0x55995f279850:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f275e40:op=0x55995f279850:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98dffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98cff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98cff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-53 info.flags=0x0 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98cff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-53 to cache LRU end 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13): adding source pipe: {s={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13): adding dest pipe: {s={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98cff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98e7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98e7fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-50 info.flags=0x0 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98e7fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-50 to cache LRU end 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10): adding source pipe: {s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10): adding dest pipe: {s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 08e8c72927a9559e5b84f7d7589436d74e5829fd860dcf41503c23354706b53e 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114604Z 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:08e8c72927a9559e5b84f7d7589436d74e5829fd860dcf41503c23354706b53e 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 57e31c9797e5e0f423226fe482aa0eb42e17dc5e368d87d8bc338dc05fe3a80c 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=57e31c9797e5e0f423226fe482aa0eb42e17dc5e368d87d8bc338dc05fe3a80c 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114604Z 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=0, curl_handle=0x55995f27e940 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 run: stack=0x55995f275e40 is io blocked 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f27a6f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f279fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f27a6f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f279fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f27a6f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f279fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f27a6f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f279fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2772b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f275fe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f27aa70:op=0x55995f27a450:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb926ffd640 20 link_request req_data=0x55995f27f550 req_data->id=0, curl_handle=0x55995f27e940 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2772b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 run: stack=0x55995f274bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f281660:op=0x55995f280fd0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f275fe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f27aa70:op=0x55995f281810:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f281660:op=0x55995f281e40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f27aa70:op=0x55995f281810:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 355fc4d4bf21ffb6f48b3f73e5f896c81cf2312805523569a4afacde70376baf 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114604Z 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:355fc4d4bf21ffb6f48b3f73e5f896c81cf2312805523569a4afacde70376baf 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 85cc3a5d1d7f6a4c645d08d41707dad54c8e938ca4cb02192f326c709e121471 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=85cc3a5d1d7f6a4c645d08d41707dad54c8e938ca4cb02192f326c709e121471 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114604Z 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=1, curl_handle=0x55995f283a50 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 run: stack=0x55995f27aa70 is io blocked 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f281660:op=0x55995f281e40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:04.732 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 2c1a7224bf53977089a3bb2de841f13c84792d4eb5f51f2da9dd82b0af70789c 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114604Z 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2c1a7224bf53977089a3bb2de841f13c84792d4eb5f51f2da9dd82b0af70789c 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = c2629ec220471d064e30ff85d2cf536dc552d93972084d2a8be7d49788a46503 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c2629ec220471d064e30ff85d2cf536dc552d93972084d2a8be7d49788a46503 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114604Z 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb98e7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=2, curl_handle=0x55995f287050 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 run: stack=0x55995f281660 is io blocked 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb926ffd640 20 link_request req_data=0x55995f2841a0 req_data->id=1, curl_handle=0x55995f283a50 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb926ffd640 20 link_request req_data=0x55995f2877a0 req_data->id=2, curl_handle=0x55995f287050 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f27b110:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f27b110:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f27b110:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f27b110:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f278580:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f289790:op=0x55995f289170:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f278580:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 run: stack=0x55995f273080 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f289790:op=0x55995f289920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 cr:s=0x55995f289790:op=0x55995f289920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.733 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.734 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.734 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.734 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.734 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:04.734 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.734 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = ab3a28ecf6c6ff92f4d69ee48e9d8fbdabda9ce7f0ac76bf08b8ccbe387768db 2026-03-20T11:46:04.734 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:04.734 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114604Z 2026-03-20T11:46:04.734 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:04.734 INFO:teuthology.orchestra.run.vm02.stderr:ab3a28ecf6c6ff92f4d69ee48e9d8fbdabda9ce7f0ac76bf08b8ccbe387768db 2026-03-20T11:46:04.734 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:04.734 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:04.734 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:04.734 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:04.734 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 2056e367df13b35eff4c3f8724699b4fc2b65e106352403c151e55c246666da1 2026-03-20T11:46:04.734 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2056e367df13b35eff4c3f8724699b4fc2b65e106352403c151e55c246666da1 2026-03-20T11:46:04.734 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.734 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114604Z 2026-03-20T11:46:04.734 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:04.734 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=3, curl_handle=0x55995f28b500 2026-03-20T11:46:04.734 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb9cb610900 20 run: stack=0x55995f289790 is io blocked 2026-03-20T11:46:04.734 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.730+0000 7fb926ffd640 20 link_request req_data=0x55995f28bc10 req_data->id=3, curl_handle=0x55995f28b500 2026-03-20T11:46:04.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 cr:s=0x55995f27aa70:op=0x55995f281810:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 cr:s=0x55995f27aa70:op=0x55995f281810:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:04.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 cr:s=0x55995f27aa70:op=0x55995f27a450:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 15 stack 0x55995f27aa70 end 2026-03-20T11:46:04.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 run: stack=0x55995f27aa70 is done 2026-03-20T11:46:04.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2772b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 collect(): s=0x55995f274bf0 stack=0x55995f27aa70 is complete 2026-03-20T11:46:04.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:04.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f281810:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:04.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.0 2026-03-20T11:46:04.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1 2026-03-20T11:46:04.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.2 2026-03-20T11:46:04.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.3 2026-03-20T11:46:04.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.4 2026-03-20T11:46:04.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.5 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.6 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.7 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.8 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.9 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.10 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 run: stack=0x55995f274bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 cr:s=0x55995f27aa70:op=0x55995f282470:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 cr:s=0x55995f27a4b0:op=0x55995f28d390:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 cr:s=0x55995f28df70:op=0x55995f28d980:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f28e220:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 cr:s=0x55995f28f910:op=0x55995f28ef20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f28fc40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f290960:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 cr:s=0x55995f292070:op=0x55995f291680:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f2923a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f2930a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f293df0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 cr:s=0x55995f27aa70:op=0x55995f282470:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 run: stack=0x55995f27aa70 is io blocked 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 cr:s=0x55995f27a4b0:op=0x55995f28d390:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 run: stack=0x55995f27a4b0 is io blocked 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 cr:s=0x55995f28df70:op=0x55995f28d980:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 run: stack=0x55995f28df70 is io blocked 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f28e220:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 run: stack=0x55995f28ec10 is io blocked 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 cr:s=0x55995f28f910:op=0x55995f28ef20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 run: stack=0x55995f28f910 is io blocked 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f28fc40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 run: stack=0x55995f290630 is io blocked 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f290960:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 run: stack=0x55995f291350 is io blocked 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 cr:s=0x55995f292070:op=0x55995f291680:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 run: stack=0x55995f292070 is io blocked 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f2923a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 run: stack=0x55995f292d90 is io blocked 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f2930a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 run: stack=0x55995f293a90 is io blocked 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f293df0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.732+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is io blocked 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.734+0000 7fb9cb610900 20 cr:s=0x55995f289790:op=0x55995f289920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 cr:s=0x55995f289790:op=0x55995f289920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 cr:s=0x55995f289790:op=0x55995f289920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 cr:s=0x55995f289790:op=0x55995f289920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 cr:s=0x55995f289790:op=0x55995f289170:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 15 stack 0x55995f289790 end 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 run: stack=0x55995f289790 is done 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f278580:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 collect(): s=0x55995f273080 stack=0x55995f289790 is complete 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:04.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f278580:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 15 stack 0x55995f273080 end 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 stack->operate() returned ret=-22 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 run: stack=0x55995f273080 is done 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f278580:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 4 starting trim on bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2898d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2898d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 enqueued request req=0x55995f27c6a0 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 req: 0x55995f27c6a0 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 run: stack=0x55995f273080 is io blocked 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb983fff640 20 dequeued request req=0x55995f27c6a0 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb983fff640 20 RGWWQ: empty 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb983fff640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 : miss 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb983fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 cr:s=0x55995f28df70:op=0x55995f28d980:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 cr:s=0x55995f28df70:op=0x55995f28d980:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 cr:s=0x55995f28df70:op=0x55995f28d980:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 cr:s=0x55995f28df70:op=0x55995f28d980:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 15 stack 0x55995f28df70 end 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 run: stack=0x55995f28df70 is done 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f281810:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 run: stack=0x55995f274bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 cr:s=0x55995f275e40:op=0x55995f279850:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 cr:s=0x55995f275e40:op=0x55995f279850:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.735+0000 7fb9cb610900 20 cr:s=0x55995f281660:op=0x55995f281e40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f28e220:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f275e40:op=0x55995f279850:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f281660:op=0x55995f281e40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f27aa70:op=0x55995f282470:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f28fc40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f293df0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f27a4b0:op=0x55995f28d390:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f2930a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f2923a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f28e220:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f275e40:op=0x55995f279850:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f281660:op=0x55995f281e40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f27aa70:op=0x55995f282470:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f28fc40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f293df0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f27a4b0:op=0x55995f28d390:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f2930a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f2923a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f28e220:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f275e40:op=0x55995f27cac0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 15 stack 0x55995f275e40 end 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f275e40 is done 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f281660:op=0x55995f281e40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f27aa70:op=0x55995f282470:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f28fc40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f293df0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f27a4b0:op=0x55995f28d390:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f2930a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f2923a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f28e220:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 15 stack 0x55995f28ec10 end 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f28ec10 is done 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f294be0:op=0x55995f271db0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 collect(): s=0x55995f294be0 stack=0x55995f275e40 is complete 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 10 trimming bilogs for bucket=:dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) markers=[00000000004.65.1,00000000002.57.1,00000000004.95.1,00000000002.36.1,00000000004.41.1,00000000002.67.1,00000000002.53.1,00000000004.82.1,00000000002.67.1,00000000002.123.1,00000000002.39.1], shards=11 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f281660:op=0x55995f280fd0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 15 stack 0x55995f281660 end 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f281660 is done 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f27aa70:op=0x55995f282470:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 15 stack 0x55995f27aa70 end 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f27aa70 is done 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f28fc40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 15 stack 0x55995f290630 end 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f290630 is done 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f293df0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 15 stack 0x55995f2947e0 end 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is done 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f27a4b0:op=0x55995f28d390:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 15 stack 0x55995f27a4b0 end 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f27a4b0 is done 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f2930a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 15 stack 0x55995f293a90 end 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f293a90 is done 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f2923a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 15 stack 0x55995f292d90 end 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f292d90 is done 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f281810:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f294be0:op=0x55995f28e220:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 10 trimming bilog shard 0 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.65.1 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 10 trimming bilog shard 1 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.57.1 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 10 trimming bilog shard 2 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.95.1 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 10 trimming bilog shard 3 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.36.1 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 10 trimming bilog shard 4 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.41.1 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 10 trimming bilog shard 5 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.67.1 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 10 trimming bilog shard 6 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.53.1 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 10 trimming bilog shard 7 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.82.1 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 10 trimming bilog shard 8 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.67.1 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 10 trimming bilog shard 9 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.123.1 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 10 trimming bilog shard 10 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.39.1 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f294be0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f275fe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 collect(): s=0x55995f18b1a0 stack=0x55995f281660 is complete 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 10 trimming bilogs for bucket=:dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) markers=[00000000004.87.6,00000000002.126.1,00000000004.75.6,00000000002.34.1,00000000004.89.6,00000000002.87.1,00000000002.128.1,00000000004.94.6,00000000002.37.1,00000000002.73.1,00000000002.61.1], shards=11 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f281810:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:04.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f27aa70:op=0x55995f285b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f275e40:op=0x55995f2c03b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f28df70:op=0x55995ee08040:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f289790:op=0x55995f293060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f2dc3a0:op=0x55995f293d80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f2e3c40:op=0x55995f2921f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f2db950:op=0x55995f2c0c80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995ee0b540:op=0x55995f281e00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f2dad60:op=0x55995f28d2b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f2be9a0:op=0x55995f2de7e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995ee0b380:op=0x55995f2bf4d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bfcd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 10 trimming bilog shard 0 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.87.6 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 10 trimming bilog shard 1 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.126.1 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 10 trimming bilog shard 2 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.75.6 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 10 trimming bilog shard 3 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.34.1 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 10 trimming bilog shard 4 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.89.6 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 10 trimming bilog shard 5 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.87.1 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 10 trimming bilog shard 6 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.128.1 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 10 trimming bilog shard 7 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.94.6 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 10 trimming bilog shard 8 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.37.1 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 10 trimming bilog shard 9 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.73.1 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 10 trimming bilog shard 10 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.61.1 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f281810:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f27aa70:op=0x55995f285b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.0 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f27aa70 is io blocked 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f275e40:op=0x55995f2c03b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.1 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f275e40 is io blocked 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f28df70:op=0x55995ee08040:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.2 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f28df70 is io blocked 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f289790:op=0x55995f293060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.3 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f289790 is io blocked 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f2dc3a0:op=0x55995f293d80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.4 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f2dc3a0 is io blocked 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f2e3c40:op=0x55995f2921f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.5 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f2e3c40 is io blocked 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f2db950:op=0x55995f2c0c80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.6 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f2db950 is io blocked 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995ee0b540:op=0x55995f281e00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.7 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995ee0b540 is io blocked 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f2dad60:op=0x55995f28d2b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.8 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f2dad60 is io blocked 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f2be9a0:op=0x55995f2de7e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.9 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f2be9a0 is io blocked 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995ee0b380:op=0x55995f2bf4d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.10 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995ee0b380 is io blocked 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f27a4b0:op=0x55995f27cac0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f281660:op=0x55995f2e10d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f289610:op=0x55995f2e18d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f286e10:op=0x55995f2df620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f2c3c10:op=0x55995f2dfe20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f28b350:op=0x55995f2e2430:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f2dc560:op=0x55995f2e2c30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f2e3e80:op=0x55995f2c2090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f2e4780:op=0x55995f2c2890:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f2875c0:op=0x55995f2c3090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f27f370:op=0x55995f2d6340:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f281810:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f27a4b0:op=0x55995f27cac0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.0 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f27a4b0 is io blocked 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f281660:op=0x55995f2e10d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.1 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f281660 is io blocked 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f289610:op=0x55995f2e18d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.2 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f289610 is io blocked 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f286e10:op=0x55995f2df620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.3 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f286e10 is io blocked 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f2c3c10:op=0x55995f2dfe20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.4 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f2c3c10 is io blocked 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f28b350:op=0x55995f2e2430:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.5 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f28b350 is io blocked 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f2dc560:op=0x55995f2e2c30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.6 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f2dc560 is io blocked 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f2e3e80:op=0x55995f2c2090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.7 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f2e3e80 is io blocked 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f2e4780:op=0x55995f2c2890:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.8 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f2e4780 is io blocked 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f2875c0:op=0x55995f2c3090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.9 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f2875c0 is io blocked 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f27f370:op=0x55995f2d6340:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.10 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f27f370 is io blocked 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f281810:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f290960:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f281810:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f290960:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f281810:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f274bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f290960:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f290960:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 15 stack 0x55995f291350 end 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f291350 is done 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f281810:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb9cb610900 20 run: stack=0x55995f274bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb983fff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb983fff640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 info.flags=0x17 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb983fff640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 to cache LRU end 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb983fff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb983fff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:04.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb983fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:46:04.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.736+0000 7fb983fff640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-49 : miss 2026-03-20T11:46:04.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.737+0000 7fb983fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f2e3c40:op=0x55995f2921f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f2e3c40:op=0x55995f2921f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995ee0b540:op=0x55995f281e00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f2dad60:op=0x55995f28d2b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f2e3c40:op=0x55995f2921f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995ee0b540:op=0x55995f281e00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f2dad60:op=0x55995f28d2b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f2e3c40:op=0x55995f2921f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 15 stack 0x55995f2e3c40 end 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 run: stack=0x55995f2e3c40 is done 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995ee0b540:op=0x55995f281e00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f2dad60:op=0x55995f28d2b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f294be0:op=0x55995f28e220:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 run: stack=0x55995f294be0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995ee0b540:op=0x55995f281e00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 15 stack 0x55995ee0b540 end 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 run: stack=0x55995ee0b540 is done 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f2dad60:op=0x55995f28d2b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 15 stack 0x55995f2dad60 end 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 run: stack=0x55995f2dad60 is done 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f294be0:op=0x55995f28e220:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f294be0:op=0x55995f28e220:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 run: stack=0x55995f294be0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995ee0b380:op=0x55995f2bf4d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995ee0b380:op=0x55995f2bf4d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995ee0b380:op=0x55995f2bf4d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995ee0b380:op=0x55995f2bf4d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 15 stack 0x55995ee0b380 end 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 run: stack=0x55995ee0b380 is done 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f294be0:op=0x55995f28e220:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 run: stack=0x55995f294be0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f289610:op=0x55995f2e18d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f289610:op=0x55995f2e18d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f289610:op=0x55995f2e18d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f289610:op=0x55995f2e18d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 15 stack 0x55995f289610 end 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 run: stack=0x55995f289610 is done 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bfcd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f28df70:op=0x55995ee08040:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f28df70:op=0x55995ee08040:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f286e10:op=0x55995f2df620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f28b350:op=0x55995f2e2430:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f28df70:op=0x55995ee08040:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f286e10:op=0x55995f2df620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f28b350:op=0x55995f2e2430:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f28df70:op=0x55995ee08040:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 15 stack 0x55995f28df70 end 2026-03-20T11:46:04.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 run: stack=0x55995f28df70 is done 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f2e4780:op=0x55995f2c2890:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f2be9a0:op=0x55995f2de7e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f2e3e80:op=0x55995f2c2090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f281660:op=0x55995f2e10d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f286e10:op=0x55995f2df620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f28b350:op=0x55995f2e2430:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f294be0:op=0x55995f28e220:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 run: stack=0x55995f294be0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f2e4780:op=0x55995f2c2890:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f2be9a0:op=0x55995f2de7e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f2e3e80:op=0x55995f2c2090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f281660:op=0x55995f2e10d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f286e10:op=0x55995f2df620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 15 stack 0x55995f286e10 end 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 run: stack=0x55995f286e10 is done 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f28b350:op=0x55995f2e2430:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 15 stack 0x55995f28b350 end 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 run: stack=0x55995f28b350 is done 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f2e4780:op=0x55995f2c2890:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f2be9a0:op=0x55995f2de7e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f2e3e80:op=0x55995f2c2090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f281660:op=0x55995f2e10d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bfcd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f2e4780:op=0x55995f2c2890:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 15 stack 0x55995f2e4780 end 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 run: stack=0x55995f2e4780 is done 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 cr:s=0x55995f2be9a0:op=0x55995f2de7e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 15 stack 0x55995f2be9a0 end 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.739+0000 7fb9cb610900 20 run: stack=0x55995f2be9a0 is done 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f2e3e80:op=0x55995f2c2090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 15 stack 0x55995f2e3e80 end 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 run: stack=0x55995f2e3e80 is done 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f281660:op=0x55995f2e10d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 15 stack 0x55995f281660 end 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 run: stack=0x55995f281660 is done 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bfcd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f294be0:op=0x55995f28e220:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 run: stack=0x55995f294be0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f27a4b0:op=0x55995f27cac0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f28f910:op=0x55995f28ef20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f292070:op=0x55995f291680:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f2db950:op=0x55995f2c0c80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f27aa70:op=0x55995f285b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f2875c0:op=0x55995f2c3090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f27f370:op=0x55995f2d6340:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f275e40:op=0x55995f2c03b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bfcd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f27a4b0:op=0x55995f27cac0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f28f910:op=0x55995f28ef20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f292070:op=0x55995f291680:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f2db950:op=0x55995f2c0c80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f27aa70:op=0x55995f285b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f2875c0:op=0x55995f2c3090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f27f370:op=0x55995f2d6340:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f275e40:op=0x55995f2c03b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bfcd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f27a4b0:op=0x55995f27cac0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f28f910:op=0x55995f28ef20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f292070:op=0x55995f291680:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f2db950:op=0x55995f2c0c80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f27aa70:op=0x55995f285b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f2875c0:op=0x55995f2c3090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f27f370:op=0x55995f2d6340:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f275e40:op=0x55995f2c03b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bfcd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f27a4b0:op=0x55995f27cac0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 15 stack 0x55995f27a4b0 end 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 run: stack=0x55995f27a4b0 is done 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f28f910:op=0x55995f28ef20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 15 stack 0x55995f28f910 end 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 run: stack=0x55995f28f910 is done 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f292070:op=0x55995f291680:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 15 stack 0x55995f292070 end 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 run: stack=0x55995f292070 is done 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f2db950:op=0x55995f2c0c80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 15 stack 0x55995f2db950 end 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 run: stack=0x55995f2db950 is done 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f27aa70:op=0x55995f285b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 15 stack 0x55995f27aa70 end 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 run: stack=0x55995f27aa70 is done 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f2875c0:op=0x55995f2c3090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 15 stack 0x55995f2875c0 end 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 run: stack=0x55995f2875c0 is done 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f27f370:op=0x55995f2d6340:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 15 stack 0x55995f27f370 end 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 run: stack=0x55995f27f370 is done 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f275e40:op=0x55995f2c03b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 15 stack 0x55995f275e40 end 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 run: stack=0x55995f275e40 is done 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bfcd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f281810:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f294be0:op=0x55995f28e220:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bfcd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f281810:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f294be0:op=0x55995f28e220:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bfcd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2772b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f294be0:op=0x55995f28e220:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 run: stack=0x55995f294be0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f290960:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f290960:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 enqueued request req=0x55995f2750c0 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 req: 0x55995f2750c0 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 run: stack=0x55995f274bf0 is io blocked 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9837fe640 20 dequeued request req=0x55995f2750c0 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9837fe640 20 RGWWQ: empty 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb983fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb983fff640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-49 info.flags=0x0 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb983fff640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-49 to cache LRU end 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb983fff640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-49 : miss 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb983fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f2dc3a0:op=0x55995f293d80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f2dc3a0:op=0x55995f293d80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f2dc3a0:op=0x55995f293d80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f2dc3a0:op=0x55995f293d80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 15 stack 0x55995f2dc3a0 end 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 run: stack=0x55995f2dc3a0 is done 2026-03-20T11:46:04.745 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 cr:s=0x55995f294be0:op=0x55995f28e220:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.740+0000 7fb9cb610900 20 run: stack=0x55995f294be0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f2dc560:op=0x55995f2e2c30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f2dc560:op=0x55995f2e2c30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f289790:op=0x55995f293060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f2c3c10:op=0x55995f2dfe20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f2dc560:op=0x55995f2e2c30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f289790:op=0x55995f293060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f2c3c10:op=0x55995f2dfe20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f2dc560:op=0x55995f2e2c30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 15 stack 0x55995f2dc560 end 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 run: stack=0x55995f2dc560 is done 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f289790:op=0x55995f293060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f2c3c10:op=0x55995f2dfe20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bfcd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f289790:op=0x55995f293060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 15 stack 0x55995f289790 end 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 run: stack=0x55995f289790 is done 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f2c3c10:op=0x55995f2dfe20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 15 stack 0x55995f2c3c10 end 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 run: stack=0x55995f2c3c10 is done 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f294be0:op=0x55995f28e220:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bfcd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f294be0:op=0x55995f271db0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 trim: trimmed bucket instance dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 15 stack 0x55995f294be0 end 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 run: stack=0x55995f294be0 is done 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f275fe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 trim: trimmed bucket instance dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 15 stack 0x55995f18b1a0 end 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is done 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995ee0b540:op=0x55995f275fe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 4 starting trim on bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995ee0b540:op=0x55995f2e0680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f271d80:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 4 starting trim on bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995ee0b540:op=0x55995f2e0680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 enqueued request req=0x55995f2dbd10 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 req: 0x55995f2dbd10 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 run: stack=0x55995ee0b540 is io blocked 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2c03b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2c03b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 enqueued request req=0x55995f27bac0 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 req: 0x55995f2dbd10 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 req: 0x55995f27bac0 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is io blocked 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb982ffd640 20 dequeued request req=0x55995f2dbd10 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb982ffd640 20 RGWWQ: 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb982ffd640 20 req: 0x55995f27bac0 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb982ffd640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 : miss 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb982ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb983fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb983fff640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-49 info.flags=0x0 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb983fff640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-49 to cache LRU end 2026-03-20T11:46:04.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb983fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9): adding source pipe: {s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9827fc640 20 dequeued request req=0x55995f27bac0 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9827fc640 20 RGWWQ: empty 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9827fc640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 : miss 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9827fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb983fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9): adding dest pipe: {s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb983fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb983fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb983fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2898d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2898d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2898d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2898d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f278580:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f279850:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f278580:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 run: stack=0x55995f273080 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995ee0a020:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995ee0a020:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 1d3c24d089a7a06c69084fb93c592cae5fb609d879dd99457a2bbf8f236394d5 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114604Z 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:1d3c24d089a7a06c69084fb93c592cae5fb609d879dd99457a2bbf8f236394d5 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = cae5b8be036a7925cf3a4a47ca79f41a28e5d8a0a5f0c4370650894ec157e30a 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=cae5b8be036a7925cf3a4a47ca79f41a28e5d8a0a5f0c4370650894ec157e30a 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114604Z 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=4, curl_handle=0x55995f287050 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb9cb610900 20 run: stack=0x55995f291350 is io blocked 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.741+0000 7fb926ffd640 20 link_request req_data=0x55995f2877a0 req_data->id=4, curl_handle=0x55995f287050 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb982ffd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb982ffd640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 info.flags=0x17 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb982ffd640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 to cache LRU end 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb982ffd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:04.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb982ffd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb982ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb982ffd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-51 : miss 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb982ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9827fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9827fc640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 info.flags=0x17 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9827fc640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 to cache LRU end 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9827fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9827fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9827fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9827fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-52 : miss 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9827fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9837fe640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 info.flags=0x11 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9837fe640 10 rgw async rados processor: moving a2.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 to cache LRU end 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9837fe640 10 rgw async rados processor: distributing notification oid=a2.rgw.control:notify.6 cni=[op: 0, obj: a2.rgw.meta:root:.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20, ofs0, ns] 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb982ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb982ffd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-51 info.flags=0x0 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb982ffd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-51 to cache LRU end 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb982ffd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-51 : miss 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb982ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb982ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb982ffd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-51 info.flags=0x0 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb982ffd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-51 to cache LRU end 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9827fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb982ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11): adding source pipe: {s={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9827fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-52 info.flags=0x0 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9827fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-52 to cache LRU end 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb982ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11): adding dest pipe: {s={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9827fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-52 : miss 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9827fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb982ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb982ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb982ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 20 cr:s=0x55995ee0b540:op=0x55995f2e0680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 20 cr:s=0x55995ee0b540:op=0x55995f2e0680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 20 cr:s=0x55995ee0b540:op=0x55995f2e0680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 20 cr:s=0x55995ee0b540:op=0x55995f2e0680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 20 cr:s=0x55995ee0b540:op=0x55995f275fe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f27df40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 20 cr:s=0x55995ee0b540:op=0x55995f275fe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 20 run: stack=0x55995ee0b540 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f30e170:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f30e170:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:04.748 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 0c2c3fc5fa7d36071e56570881bd0e8dca1db514539f42b896a7ee47a2595775 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114604Z 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:0c2c3fc5fa7d36071e56570881bd0e8dca1db514539f42b896a7ee47a2595775 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = ffdf43e7afb5a4390f5a254bee69910b016045e2d3cadc1dc416ee5b9fc5146d 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=ffdf43e7afb5a4390f5a254bee69910b016045e2d3cadc1dc416ee5b9fc5146d 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114604Z 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=5, curl_handle=0x55995f27e940 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is io blocked 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb926ffd640 20 link_request req_data=0x55995f27f550 req_data->id=5, curl_handle=0x55995f27e940 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9a27fc640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 343597383931 cookie 94117215045808 notifier 6376 bl.length()=567 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9827fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9a27fc640 10 rgw watcher librados: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 info.flags=0x11 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9a27fc640 10 rgw watcher librados: moving a2.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 to cache LRU end 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9827fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-52 info.flags=0x0 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.742+0000 7fb9827fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-52 to cache LRU end 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9827fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12): adding source pipe: {s={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9827fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12): adding dest pipe: {s={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9827fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9827fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9827fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2c03b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2c03b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2c03b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2c03b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f271d80:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f2bebf0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f271d80:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f2dd960:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f2dd960:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 7b749aa0cebf65e3c60fbd244945b0eade8a4c20e6fc612560e04ad8d968a689 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114604Z 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:7b749aa0cebf65e3c60fbd244945b0eade8a4c20e6fc612560e04ad8d968a689 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = efcf3137948f6340926a92a56c8dbe3f4a1c59bfebe23d6b27cf77f5547c0dd1 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=efcf3137948f6340926a92a56c8dbe3f4a1c59bfebe23d6b27cf77f5547c0dd1 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114604Z 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=6, curl_handle=0x55995f28b500 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 20 run: stack=0x55995f293a90 is io blocked 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb926ffd640 20 link_request req_data=0x55995f28bc10 req_data->id=6, curl_handle=0x55995f28b500 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9837fe640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-60[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20]): orig_sources= new_sources= 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9837fe640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-60[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20]): potential sources added=[] removed=[] 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9837fe640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-60[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20]): orig_dests= new_dests= 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9837fe640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-60[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20]): potential dests added=[] removed=[] 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.743+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f290960:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.744+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f290960:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.744+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f290960:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.744+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f290960:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.744+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2772b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.744+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2dbf80:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.744+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2dbf80:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.744+0000 7fb9cb610900 20 enqueued request req=0x55995ed93f40 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.744+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.744+0000 7fb9cb610900 20 req: 0x55995ed93f40 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.744+0000 7fb9cb610900 20 run: stack=0x55995f274bf0 is io blocked 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.744+0000 7fb981ffb640 20 dequeued request req=0x55995ed93f40 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.744+0000 7fb981ffb640 20 RGWWQ: empty 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.745+0000 7fb981ffb640 10 rgw async rados processor: removing a2.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 from cache 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.745+0000 7fb981ffb640 10 rgw async rados processor: distributing notification oid=a2.rgw.control:notify.6 cni=[op: 1, obj: a2.rgw.meta:root:.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20, ofs0, ns] 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.745+0000 7fb9a2ffd640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 343597383932 cookie 94117215045808 notifier 6376 bl.length()=206 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2dbf80:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2dbf80:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2dbf80:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2dbf80:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2772b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9cb610900 20 trim: trimmed bucket instance dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9cb610900 15 stack 0x55995f274bf0 end 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9cb610900 20 run: stack=0x55995f274bf0 is done 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2772b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9cb610900 4 starting trim on bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2c03b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2c03b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9cb610900 20 enqueued request req=0x55995f290f50 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9cb610900 20 req: 0x55995f290f50 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9cb610900 20 run: stack=0x55995f274bf0 is io blocked 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9817fa640 20 dequeued request req=0x55995f290f50 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9817fa640 20 RGWWQ: empty 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9817fa640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 : miss 2026-03-20T11:46:04.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9817fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9817fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9817fa640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 info.flags=0x17 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9817fa640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 to cache LRU end 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9817fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9817fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9817fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9817fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-58 : miss 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9817fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9817fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9817fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-58 info.flags=0x0 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9817fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-58 to cache LRU end 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9817fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-58 : miss 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.746+0000 7fb9817fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9817fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9817fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-58 info.flags=0x0 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9817fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-58 to cache LRU end 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9817fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18): adding source pipe: {s={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9817fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18): adding dest pipe: {s={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9817fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9817fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9817fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2c03b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2c03b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2c03b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2c03b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2772b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995ee09450:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2772b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 20 run: stack=0x55995f274bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f287070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f287070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.751 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:04.752 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:04.752 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:04.752 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.752 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.752 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.752 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.752 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.752 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:04.752 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 6e0543b61304b8746153395ed0de55d493e8a0d57e17f22bb4ee72113aa6195d 2026-03-20T11:46:04.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:04.752 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114604Z 2026-03-20T11:46:04.752 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:04.752 INFO:teuthology.orchestra.run.vm02.stderr:6e0543b61304b8746153395ed0de55d493e8a0d57e17f22bb4ee72113aa6195d 2026-03-20T11:46:04.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:04.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:04.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:04.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:04.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 2a617e37721800c8484120d7ff274173ae44b1f51c4daf1b8f8a42949a6a6b0a 2026-03-20T11:46:04.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2a617e37721800c8484120d7ff274173ae44b1f51c4daf1b8f8a42949a6a6b0a 2026-03-20T11:46:04.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114604Z 2026-03-20T11:46:04.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:04.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=7, curl_handle=0x55995f283a50 2026-03-20T11:46:04.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb9cb610900 20 run: stack=0x55995f292d90 is io blocked 2026-03-20T11:46:04.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.747+0000 7fb926ffd640 20 link_request req_data=0x7fb96c0062e0 req_data->id=7, curl_handle=0x55995f283a50 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995ee0a020:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995ee0a020:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995ee0a020:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995ee0a020:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f279850:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb9cb610900 15 stack 0x55995f291350 end 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb9cb610900 20 run: stack=0x55995f291350 is done 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f278580:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb9cb610900 20 collect(): s=0x55995f273080 stack=0x55995f291350 is complete 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb9cb610900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb9cb610900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f278580:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb9cb610900 15 stack 0x55995f273080 end 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb9cb610900 20 stack->operate() returned ret=-22 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb9cb610900 20 run: stack=0x55995f273080 is done 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb9cb610900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f30f8d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb9cb610900 4 starting trim on bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2e0680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2e0680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb9cb610900 20 enqueued request req=0x55995f27c6a0 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb9cb610900 20 req: 0x55995f27c6a0 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb9cb610900 20 run: stack=0x55995f273080 is io blocked 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb980ff9640 20 dequeued request req=0x55995f27c6a0 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb980ff9640 20 RGWWQ: empty 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb980ff9640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 : miss 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb980ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb980ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb980ff9640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 info.flags=0x17 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb980ff9640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 to cache LRU end 2026-03-20T11:46:04.780 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb980ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:04.781 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb980ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:04.781 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb980ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:46:04.781 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb980ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-59 : miss 2026-03-20T11:46:04.781 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.783+0000 7fb980ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.781 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.784+0000 7fb980ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.781 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.784+0000 7fb980ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-59 info.flags=0x0 2026-03-20T11:46:04.781 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.784+0000 7fb980ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-59 to cache LRU end 2026-03-20T11:46:04.781 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.784+0000 7fb980ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-59 : miss 2026-03-20T11:46:04.781 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.784+0000 7fb980ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.781 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.784+0000 7fb980ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.781 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.784+0000 7fb980ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-59 info.flags=0x0 2026-03-20T11:46:04.781 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.784+0000 7fb980ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-59 to cache LRU end 2026-03-20T11:46:04.781 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.784+0000 7fb980ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19): adding source pipe: {s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.781 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.784+0000 7fb980ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19): adding dest pipe: {s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.784+0000 7fb980ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.784+0000 7fb980ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.784+0000 7fb980ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.784+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2e0680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.784+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2e0680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.784+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2e0680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.784+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2e0680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.784+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f30f8d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.784+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995ee0a020:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.784+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f30f8d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.784+0000 7fb9cb610900 20 run: stack=0x55995f273080 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.784+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2dbf80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.784+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2dbf80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.784+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.784+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.784+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.785+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.785+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.785+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.785+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 1394a6b744ddbf7f3499cdad8b8858a76d3e55dc3deba20ddad0593456cea3b5 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.785+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114604Z 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:1394a6b744ddbf7f3499cdad8b8858a76d3e55dc3deba20ddad0593456cea3b5 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.785+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.785+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.785+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.785+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.785+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 4d2b777adb0bfdf9642f7a985832f2482a7e3fa965c7b513f411a5cd01d279f1 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.785+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=4d2b777adb0bfdf9642f7a985832f2482a7e3fa965c7b513f411a5cd01d279f1 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.785+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.785+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114604Z 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.785+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.785+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=8, curl_handle=0x55995f287050 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.785+0000 7fb9cb610900 20 run: stack=0x55995f291350 is io blocked 2026-03-20T11:46:04.782 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.785+0000 7fb926ffd640 20 link_request req_data=0x55995f2877a0 req_data->id=8, curl_handle=0x55995f287050 2026-03-20T11:46:04.783 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f30e170:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.783 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f30e170:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.783 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f2dd960:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.783 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f30e170:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.783 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f2dd960:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.783 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f30e170:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.783 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f2dd960:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.783 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f27df40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.783 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 15 stack 0x55995f2947e0 end 2026-03-20T11:46:04.783 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is done 2026-03-20T11:46:04.783 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f2dd960:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995ee0b540:op=0x55995f275fe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 collect(): s=0x55995ee0b540 stack=0x55995f2947e0 is complete 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 10 trimming bilogs for bucket=:dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) markers=[00000000002.62.6,,00000000002.48.6,,00000000002.70.6,,,00000000002.59.6,,,], shards=11 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f2bebf0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 15 stack 0x55995f293a90 end 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 run: stack=0x55995f293a90 is done 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995ee0b540:op=0x55995f30c8d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 10 trimming bilog shard 0 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.62.6 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 10 trimming bilog shard 2 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.48.6 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 10 trimming bilog shard 4 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.70.6 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 10 trimming bilog shard 7 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.59.6 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 run: stack=0x55995ee0b540 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f271d80:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 collect(): s=0x55995f18b1a0 stack=0x55995f293a90 is complete 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 10 trimming bilogs for bucket=:dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) markers=[00000000002.30.6,,00000000002.60.6,,00000000002.47.6,,,00000000002.30.6,,,], shards=11 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2bd640:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f292ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f2c0360:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995f30ee00:op=0x55995f2df620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f290960:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 10 trimming bilog shard 0 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.30.6 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 10 trimming bilog shard 2 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.60.6 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 10 trimming bilog shard 4 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.47.6 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 10 trimming bilog shard 7 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.30.6 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2bd640:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.0 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is io blocked 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f292ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.2 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 run: stack=0x55995f290630 is io blocked 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f2c0360:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.4 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 run: stack=0x55995f28ec10 is io blocked 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995f30ee00:op=0x55995f2df620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.7 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 run: stack=0x55995f30ee00 is io blocked 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f28ed50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f292070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995f2e2240:op=0x55995f2de9a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f2e23f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f28ed50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.0 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 run: stack=0x55995f293a90 is io blocked 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f292070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.2 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 run: stack=0x55995f2bfaa0 is io blocked 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995f2e2240:op=0x55995f2de9a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.4 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 run: stack=0x55995f2e2240 is io blocked 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f2e23f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.7 2026-03-20T11:46:04.784 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.786+0000 7fb9cb610900 20 run: stack=0x55995f2be050 is io blocked 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2bd640:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2bd640:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2bd640:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2bd640:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 15 stack 0x55995f2947e0 end 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is done 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 cr:s=0x55995ee0b540:op=0x55995f30c8d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 run: stack=0x55995ee0b540 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f287070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f287070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f287070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f287070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995ee09450:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 15 stack 0x55995f292d90 end 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 run: stack=0x55995f292d90 is done 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2772b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 collect(): s=0x55995f274bf0 stack=0x55995f292d90 is complete 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 10 trimming bilogs for bucket=:dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) markers=[,00000000001.98.6,,,,00000000001.102.6,,,00000000001.40.6,,00000000001.48.6], shards=11 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2d9340:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 10 trimming bilog shard 1 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.98.6 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 10 trimming bilog shard 5 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.102.6 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 10 trimming bilog shard 8 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.40.6 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 10 trimming bilog shard 10 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.48.6 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 run: stack=0x55995f274bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f2bcb10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f293bd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 cr:s=0x55995f286e10:op=0x55995edfda30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 cr:s=0x55995f28f910:op=0x55995f2c2d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f2bcb10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.1 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 run: stack=0x55995f292d90 is io blocked 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f293bd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.5 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is io blocked 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 cr:s=0x55995f286e10:op=0x55995edfda30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.8 2026-03-20T11:46:04.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 run: stack=0x55995f286e10 is io blocked 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 cr:s=0x55995f28f910:op=0x55995f2c2d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.10 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.789+0000 7fb9cb610900 20 run: stack=0x55995f28f910 is io blocked 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f2e23f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f2e23f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f2c0360:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f28ed50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f292070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f2e23f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f2c0360:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f28ed50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f292070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f2e23f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 15 stack 0x55995f2be050 end 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 20 run: stack=0x55995f2be050 is done 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f2c0360:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f28ed50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f292070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f290960:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f2c0360:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 15 stack 0x55995f28ec10 end 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 20 run: stack=0x55995f28ec10 is done 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f28ed50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 15 stack 0x55995f293a90 end 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 20 run: stack=0x55995f293a90 is done 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f292070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 15 stack 0x55995f2bfaa0 end 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 20 run: stack=0x55995f2bfaa0 is done 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 20 cr:s=0x55995ee0b540:op=0x55995f30c8d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 20 run: stack=0x55995ee0b540 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.787 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f290960:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f290960:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.790+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f293bd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f293bd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f2bcb10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f293bd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f2bcb10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f293bd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 15 stack 0x55995f2947e0 end 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is done 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f2bcb10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2d9340:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 run: stack=0x55995f274bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f2bcb10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 15 stack 0x55995f292d90 end 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 run: stack=0x55995f292d90 is done 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2d9340:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 run: stack=0x55995f274bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f292ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f292ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f292ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f292ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 15 stack 0x55995f290630 end 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 run: stack=0x55995f290630 is done 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995ee0b540:op=0x55995f30c8d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 run: stack=0x55995ee0b540 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f28f910:op=0x55995f2c2d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f28f910:op=0x55995f2c2d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f28f910:op=0x55995f2c2d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f28f910:op=0x55995f2c2d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 15 stack 0x55995f28f910 end 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 run: stack=0x55995f28f910 is done 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2d9340:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 run: stack=0x55995f274bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.788 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f30ee00:op=0x55995f2df620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f30ee00:op=0x55995f2df620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f2e2240:op=0x55995f2de9a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f30ee00:op=0x55995f2df620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f2e2240:op=0x55995f2de9a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f30ee00:op=0x55995f2df620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 15 stack 0x55995f30ee00 end 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 run: stack=0x55995f30ee00 is done 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f2e2240:op=0x55995f2de9a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995ee0b540:op=0x55995f30c8d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f2e2240:op=0x55995f2de9a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 15 stack 0x55995f2e2240 end 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 run: stack=0x55995f2e2240 is done 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995ee0b540:op=0x55995f275fe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 trim: trimmed bucket instance dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 15 stack 0x55995ee0b540 end 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 run: stack=0x55995ee0b540 is done 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f290960:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f271d80:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 trim: trimmed bucket instance dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 15 stack 0x55995f18b1a0 end 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is done 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f275fe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 4 starting trim on bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e0680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f271d60:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 4 starting trim on bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e0680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 enqueued request req=0x55995f283d90 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 req: 0x55995f283d90 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9807f8640 20 dequeued request req=0x55995f283d90 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9807f8640 20 RGWWQ: empty 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9807f8640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 : miss 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9807f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 run: stack=0x55995f290630 is io blocked 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f30f070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f30f070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 enqueued request req=0x55995f2dacc0 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 req: 0x55995f2dacc0 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb97fff7640 20 dequeued request req=0x55995f2dacc0 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb97fff7640 20 RGWWQ: empty 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb97fff7640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 : miss 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb97fff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.789 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is io blocked 2026-03-20T11:46:04.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f286e10:op=0x55995edfda30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f286e10:op=0x55995edfda30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f286e10:op=0x55995edfda30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f286e10:op=0x55995edfda30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 15 stack 0x55995f286e10 end 2026-03-20T11:46:04.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 run: stack=0x55995f286e10 is done 2026-03-20T11:46:04.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2d9340:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2772b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 trim: trimmed bucket instance dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:04.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 15 stack 0x55995f274bf0 end 2026-03-20T11:46:04.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 run: stack=0x55995f274bf0 is done 2026-03-20T11:46:04.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:04.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.790 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 4 starting trim on bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f30c8d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f30c8d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 enqueued request req=0x55995f30d0d0 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.791+0000 7fb9cb610900 20 req: 0x55995f30d0d0 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9807f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9807f8640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 info.flags=0x17 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9807f8640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 to cache LRU end 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9807f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9807f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9807f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9807f8640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-57 : miss 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9807f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97fff7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97fff7640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 info.flags=0x17 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97fff7640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 to cache LRU end 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97fff7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97fff7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97fff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97fff7640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-55 : miss 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97fff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97f7f6640 20 dequeued request req=0x55995f30d0d0 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97f7f6640 20 RGWWQ: empty 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97f7f6640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 : miss 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97fff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97fff7640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-55 info.flags=0x0 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97fff7640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-55 to cache LRU end 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97fff7640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-55 : miss 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97fff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9807f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9807f8640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-57 info.flags=0x0 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9807f8640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-57 to cache LRU end 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9807f8640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-57 : miss 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9807f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97fff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97fff7640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-55 info.flags=0x0 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97fff7640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-55 to cache LRU end 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97fff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding source pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97fff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding dest pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97fff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97fff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97fff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97f7f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97f7f6640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 info.flags=0x17 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97f7f6640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 to cache LRU end 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97f7f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97f7f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97f7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97f7f6640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-28 : miss 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9807f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9807f8640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-57 info.flags=0x0 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9807f8640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-57 to cache LRU end 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9807f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17): adding source pipe: {s={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9807f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17): adding dest pipe: {s={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9807f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.791 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9807f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9807f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9cb610900 20 run: stack=0x55995f274bf0 is io blocked 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f30f070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e0680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f30f070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e0680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f30f070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e0680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f30f070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e0680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f271d60:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f275fe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f27df40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f271d60:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2bc2e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f275fe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9cb610900 20 run: stack=0x55995f290630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f287070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2bf450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f287070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97f7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97f7f6640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-28 info.flags=0x0 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97f7f6640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-28 to cache LRU end 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97f7f6640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-28 : miss 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.792+0000 7fb97f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 93076512cebb4e840568f83b419d77aa0508632cb1a2f8bc97058198b6570d16 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114604Z 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:93076512cebb4e840568f83b419d77aa0508632cb1a2f8bc97058198b6570d16 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = e7686082e443207d32517cb72c22d8be971025709b66f90df2fa2123243c0d70 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=e7686082e443207d32517cb72c22d8be971025709b66f90df2fa2123243c0d70 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114604Z 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:04.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=9, curl_handle=0x55995f283a50 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 20 run: stack=0x55995f292d90 is io blocked 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2bf450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb926ffd640 20 link_request req_data=0x7fb96c0062e0 req_data->id=9, curl_handle=0x55995f283a50 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 20f487608e1a487be29e7fd4a7e7bb95c295dc0622ce338301667474692dd673 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114604Z 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:20f487608e1a487be29e7fd4a7e7bb95c295dc0622ce338301667474692dd673 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 272bd150dec2f2aed9f52fecb709226fe324965effa4ace8520f33ebd1b9e0be 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=272bd150dec2f2aed9f52fecb709226fe324965effa4ace8520f33ebd1b9e0be 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114604Z 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=10, curl_handle=0x55995f28b500 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is io blocked 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.793+0000 7fb926ffd640 20 link_request req_data=0x55995f28bc10 req_data->id=10, curl_handle=0x55995f28b500 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb97f7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb97f7f6640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-28 info.flags=0x0 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb97f7f6640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-28 to cache LRU end 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb97f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding source pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb97f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding dest pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb97f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb97f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb97f7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f30c8d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f30c8d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f30c8d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f30c8d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2dff90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 20 run: stack=0x55995f274bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995ee09450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995ee09450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 85f241d46f93da575c65c7dd15646a1a65b8e2d5e68ca0a5fb533f4bd770223e 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114604Z 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:85f241d46f93da575c65c7dd15646a1a65b8e2d5e68ca0a5fb533f4bd770223e 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 311237605a1ef5f9dfca5895ebd1d4b25d152983248681a15c73f24cdfb411ff 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=311237605a1ef5f9dfca5895ebd1d4b25d152983248681a15c73f24cdfb411ff 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114604Z 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=11, curl_handle=0x55995f27e940 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.794+0000 7fb9cb610900 20 run: stack=0x55995f2bfaa0 is io blocked 2026-03-20T11:46:04.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.795+0000 7fb926ffd640 20 link_request req_data=0x55995f27f550 req_data->id=11, curl_handle=0x55995f27e940 2026-03-20T11:46:04.794 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:04.797+0000 7f1233156640 -1 req 4666131539433196291 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2dbf80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2dbf80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2dbf80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2dbf80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995ee0a020:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 15 stack 0x55995f291350 end 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 20 run: stack=0x55995f291350 is done 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f30f8d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 20 collect(): s=0x55995f273080 stack=0x55995f291350 is complete 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 10 trimming bilogs for bucket=:dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) markers=[,00000000001.65.6,,,,00000000001.44.6,,,00000000001.58.6,,00000000001.63.6], shards=11 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f290960:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 10 trimming bilog shard 1 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.65.6 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 10 trimming bilog shard 5 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.44.6 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 10 trimming bilog shard 8 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.58.6 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 10 trimming bilog shard 10 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.63.6 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 20 run: stack=0x55995f273080 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f292070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f30d8f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f285b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f28e190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f292070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.1 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 20 run: stack=0x55995f291350 is io blocked 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f30d8f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.5 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 20 run: stack=0x55995f293a90 is io blocked 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f285b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.8 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 20 run: stack=0x55995f28ec10 is io blocked 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f28e190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.10 2026-03-20T11:46:04.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.828+0000 7fb9cb610900 20 run: stack=0x55995f2be050 is io blocked 2026-03-20T11:46:04.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.830+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f28e190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.830+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f28e190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.830+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f30d8f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.830+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f28e190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.830+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f30d8f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.830+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f28e190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.830+0000 7fb9cb610900 15 stack 0x55995f2be050 end 2026-03-20T11:46:04.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.830+0000 7fb9cb610900 20 run: stack=0x55995f2be050 is done 2026-03-20T11:46:04.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.830+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f30d8f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.830+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f290960:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.830+0000 7fb9cb610900 20 run: stack=0x55995f273080 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.828 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.830+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f30d8f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.830+0000 7fb9cb610900 15 stack 0x55995f293a90 end 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.830+0000 7fb9cb610900 20 run: stack=0x55995f293a90 is done 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.830+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f290960:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.830+0000 7fb9cb610900 20 run: stack=0x55995f273080 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f292070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f292070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f292070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f292070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 15 stack 0x55995f291350 end 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 20 run: stack=0x55995f291350 is done 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f290960:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 20 run: stack=0x55995f273080 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f285b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f285b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f285b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f285b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 15 stack 0x55995f28ec10 end 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 20 run: stack=0x55995f28ec10 is done 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f290960:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f30f8d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 20 trim: trimmed bucket instance dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 15 stack 0x55995f273080 end 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 20 run: stack=0x55995f273080 is done 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f30f8d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 4 starting trim on bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f30c8d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f30c8d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 20 enqueued request req=0x55995f27c6a0 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 20 req: 0x55995f27c6a0 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb97eff5640 20 dequeued request req=0x55995f27c6a0 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb97eff5640 20 RGWWQ: empty 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb97eff5640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 : miss 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb97eff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.831+0000 7fb9cb610900 20 run: stack=0x55995f273080 is io blocked 2026-03-20T11:46:04.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb97eff5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:04.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb97eff5640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 info.flags=0x17 2026-03-20T11:46:04.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb97eff5640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 to cache LRU end 2026-03-20T11:46:04.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb97eff5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:04.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb97eff5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:04.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb97eff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:04.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb97eff5640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-31 : miss 2026-03-20T11:46:04.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb97eff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb97eff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb97eff5640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-31 info.flags=0x0 2026-03-20T11:46:04.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb97eff5640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-31 to cache LRU end 2026-03-20T11:46:04.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb97eff5640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-31 : miss 2026-03-20T11:46:04.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb97eff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb97eff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb97eff5640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-31 info.flags=0x0 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb97eff5640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-31 to cache LRU end 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb97eff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding source pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb97eff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding dest pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb97eff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb97eff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb97eff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f30c8d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f30c8d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f30c8d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f30c8d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f30f8d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995ee0a020:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f30f8d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb9cb610900 20 run: stack=0x55995f273080 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f27d040:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f27d040:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = fbd77b7be3280f3f0c2468ec8bb013bd03c4dad99972f0b52d7b6636263f970e 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114604Z 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:fbd77b7be3280f3f0c2468ec8bb013bd03c4dad99972f0b52d7b6636263f970e 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = bc666e3d7848abdfdeb39e28fcc35cb091090d4caa5aac1a969cf31c60ffcc6c 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=bc666e3d7848abdfdeb39e28fcc35cb091090d4caa5aac1a969cf31c60ffcc6c 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114604Z 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.832+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.833+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=12, curl_handle=0x55995f287050 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.833+0000 7fb9cb610900 20 run: stack=0x55995f28ec10 is io blocked 2026-03-20T11:46:04.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.833+0000 7fb926ffd640 20 link_request req_data=0x55995f2877a0 req_data->id=12, curl_handle=0x55995f287050 2026-03-20T11:46:04.832 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:04.835+0000 7f12b0250640 -1 req 3865973014478903539 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f287070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f287070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2bf450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f287070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2bf450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f287070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2bf450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f27df40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 15 stack 0x55995f292d90 end 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 run: stack=0x55995f292d90 is done 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2bf450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f271d60:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 collect(): s=0x55995f18b1a0 stack=0x55995f292d90 is complete 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 10 trimming bilogs for bucket=:dipepu-55[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15]) markers=[,,,,,00000000001.133.6,,,,,], shards=11 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2bc2e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 15 stack 0x55995f2947e0 end 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is done 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2ddbd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 10 trimming bilog shard 5 of :dipepu-55[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15]) at marker 00000000001.133.6 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f275fe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 collect(): s=0x55995f290630 stack=0x55995f2947e0 is complete 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 10 trimming bilogs for bucket=:dipepu-57[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17]) markers=[,00000000001.63.6,,,,00000000001.43.6,,,,,], shards=11 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f2df620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2dc160:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 10 trimming bilog shard 1 of :dipepu-57[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17]) at marker 00000000001.63.6 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 10 trimming bilog shard 5 of :dipepu-57[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17]) at marker 00000000001.43.6 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 run: stack=0x55995f290630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f2df620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15.5 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 run: stack=0x55995f292d90 is io blocked 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f292ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f292ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17.1 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is io blocked 2026-03-20T11:46:04.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17.5 2026-03-20T11:46:04.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.837+0000 7fb9cb610900 20 run: stack=0x55995f291350 is io blocked 2026-03-20T11:46:04.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.838+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f2df620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.838+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f2df620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.838+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f2df620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.838+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f2df620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.838+0000 7fb9cb610900 15 stack 0x55995f292d90 end 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.838+0000 7fb9cb610900 20 run: stack=0x55995f292d90 is done 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.838+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2ddbd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.838+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f271d60:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.838+0000 7fb9cb610900 20 trim: trimmed bucket instance dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.838+0000 7fb9cb610900 15 stack 0x55995f18b1a0 end 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.838+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is done 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.838+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.838+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.838+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f271d60:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.838+0000 7fb9cb610900 4 starting trim on bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.838+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f291490:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.838+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f291490:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.838+0000 7fb9cb610900 20 enqueued request req=0x55995f283d90 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.838+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.838+0000 7fb9cb610900 20 req: 0x55995f283d90 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb97e7f4640 20 dequeued request req=0x55995f283d90 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb97e7f4640 20 RGWWQ: empty 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb97e7f4640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 : miss 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb97e7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is io blocked 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f292ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995ee09450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995ee09450:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f292ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2dff90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 15 stack 0x55995f2bfaa0 end 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 run: stack=0x55995f2bfaa0 is done 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f292ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 collect(): s=0x55995f274bf0 stack=0x55995f2bfaa0 is complete 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 15 stack 0x55995f274bf0 end 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 stack->operate() returned ret=-22 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 run: stack=0x55995f274bf0 is done 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f292ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 15 stack 0x55995f2947e0 end 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is done 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 15 stack 0x55995f291350 end 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 run: stack=0x55995f291350 is done 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2dc160:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f28d5b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 4 starting trim on bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2dc160:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2bc2e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f275fe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 trim: trimmed bucket instance dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 15 stack 0x55995f290630 end 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 run: stack=0x55995f290630 is done 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2bc2e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 enqueued request req=0x55995f293860 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb9cb610900 20 req: 0x55995f293860 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb97e7f4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb97e7f4640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 info.flags=0x17 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb97e7f4640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 to cache LRU end 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb97e7f4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb97e7f4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb97e7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb97e7f4640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-39 : miss 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb97e7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb97e7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb97e7f4640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-39 info.flags=0x0 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb97e7f4640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-39 to cache LRU end 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb97e7f4640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-39 : miss 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.839+0000 7fb97e7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb97e7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb97e7f4640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-39 info.flags=0x0 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb97e7f4640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-39 to cache LRU end 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb97e7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding source pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb97e7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding dest pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb97e7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb97e7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb97e7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb97dff3640 20 dequeued request req=0x55995f293860 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb97dff3640 20 RGWWQ: empty 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb97dff3640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 : miss 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb97dff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 run: stack=0x55995f274bf0 is io blocked 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f291490:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 4 starting trim on bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f291490:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f291490:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 enqueued request req=0x55995f30d0d0 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 req: 0x55995f30d0d0 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 run: stack=0x55995f290630 is io blocked 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f291490:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f271d60:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f27df40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f271d60:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2dcbb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2dcbb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb97d7f2640 20 dequeued request req=0x55995f30d0d0 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb97d7f2640 20 RGWWQ: empty 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb97d7f2640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 : miss 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb97d7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.840 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:04.841 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:04.841 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:04.841 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.841 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.841 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.841 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.841 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.841 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:04.841 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 98dbb5e7a302ada8b30d5b3462521943372b01c12f28959981b0ef59e6536dfc 2026-03-20T11:46:04.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:04.841 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114604Z 2026-03-20T11:46:04.841 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:04.841 INFO:teuthology.orchestra.run.vm02.stderr:98dbb5e7a302ada8b30d5b3462521943372b01c12f28959981b0ef59e6536dfc 2026-03-20T11:46:04.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:04.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:04.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:04.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:04.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = e039e44fdcbb8a1804c1339079e5bdd2138459094785a4dca2a4f9ed656b4859 2026-03-20T11:46:04.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=e039e44fdcbb8a1804c1339079e5bdd2138459094785a4dca2a4f9ed656b4859 2026-03-20T11:46:04.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114604Z 2026-03-20T11:46:04.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:04.841 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=13, curl_handle=0x55995f27e940 2026-03-20T11:46:04.843 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb9cb610900 20 run: stack=0x55995f291350 is io blocked 2026-03-20T11:46:04.843 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb97dff3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:46:04.843 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb97dff3640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 info.flags=0x17 2026-03-20T11:46:04.843 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb97dff3640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 to cache LRU end 2026-03-20T11:46:04.843 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb97dff3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:04.843 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb97dff3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:04.843 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb97dff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:04.843 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb97dff3640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-8 : miss 2026-03-20T11:46:04.843 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb97dff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.843 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.840+0000 7fb926ffd640 20 link_request req_data=0x55995f27f550 req_data->id=13, curl_handle=0x55995f27e940 2026-03-20T11:46:04.843 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.841+0000 7fb97d7f2640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:04.843 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.841+0000 7fb97d7f2640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 info.flags=0x17 2026-03-20T11:46:04.843 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.841+0000 7fb97d7f2640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 to cache LRU end 2026-03-20T11:46:04.843 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.841+0000 7fb97d7f2640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:04.843 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.841+0000 7fb97d7f2640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:04.843 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.841+0000 7fb97d7f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:04.843 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.841+0000 7fb97d7f2640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-16 : miss 2026-03-20T11:46:04.843 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.841+0000 7fb97d7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.843 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.841+0000 7fb97d7f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.843 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.841+0000 7fb97d7f2640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-16 info.flags=0x0 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.841+0000 7fb97d7f2640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-16 to cache LRU end 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.841+0000 7fb97d7f2640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-16 : miss 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.841+0000 7fb97d7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.841+0000 7fb97dff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.841+0000 7fb97dff3640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-8 info.flags=0x0 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.841+0000 7fb97dff3640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-8 to cache LRU end 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.841+0000 7fb97dff3640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-8 : miss 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.841+0000 7fb97dff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.841+0000 7fb97d7f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.841+0000 7fb97d7f2640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-16 info.flags=0x0 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.841+0000 7fb97d7f2640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-16 to cache LRU end 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.841+0000 7fb97d7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding source pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.841+0000 7fb97d7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding dest pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.841+0000 7fb97d7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.841+0000 7fb97d7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.841+0000 7fb97d7f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.842+0000 7fb97dff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.842+0000 7fb97dff3640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-8 info.flags=0x0 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.842+0000 7fb97dff3640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-8 to cache LRU end 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.842+0000 7fb97dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding source pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.842+0000 7fb97dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding dest pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.842+0000 7fb97dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.842+0000 7fb97dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.842+0000 7fb97dff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2bc2e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2bc2e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2bc2e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2bc2e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f287070:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 run: stack=0x55995f290630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f28d5b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2bebf0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2c0b60:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f28d5b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 run: stack=0x55995f274bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2bebf0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.844 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 0043f3d15cc00997e83a1a94913a1cfeb39ac6afbd6278cfcfc15428e8e38c67 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114604Z 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:0043f3d15cc00997e83a1a94913a1cfeb39ac6afbd6278cfcfc15428e8e38c67 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 17431c1cce83f7ab208aa5cc5fbdf336ec9083ca8387924d547abbcaf5e9df5f 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=17431c1cce83f7ab208aa5cc5fbdf336ec9083ca8387924d547abbcaf5e9df5f 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114604Z 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=14, curl_handle=0x55995f28b500 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is io blocked 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2bf450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2bf450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 9154b9392bd5965bae458dfcf5b95a99390d31b2d7bd6a725ccb4d6cd6e34258 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114604Z 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:9154b9392bd5965bae458dfcf5b95a99390d31b2d7bd6a725ccb4d6cd6e34258 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb926ffd640 20 link_request req_data=0x55995f28bc10 req_data->id=14, curl_handle=0x55995f28b500 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 103300e04e6de1784614b15d100a702d3270d75f161b28ed3a3427b0be2c7622 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=103300e04e6de1784614b15d100a702d3270d75f161b28ed3a3427b0be2c7622 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114604Z 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=15, curl_handle=0x55995f283a50 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb9cb610900 20 run: stack=0x55995f2bfaa0 is io blocked 2026-03-20T11:46:04.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.843+0000 7fb926ffd640 20 link_request req_data=0x7fb96c0062e0 req_data->id=15, curl_handle=0x55995f283a50 2026-03-20T11:46:04.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f27d040:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f27d040:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:04.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995ee0a020:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb9cb610900 15 stack 0x55995f28ec10 end 2026-03-20T11:46:04.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb9cb610900 20 run: stack=0x55995f28ec10 is done 2026-03-20T11:46:04.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f30f8d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb9cb610900 20 collect(): s=0x55995f273080 stack=0x55995f28ec10 is complete 2026-03-20T11:46:04.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:04.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb9cb610900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:04.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb9cb610900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:04.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f30f8d0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb9cb610900 15 stack 0x55995f273080 end 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb9cb610900 20 stack->operate() returned ret=-22 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb9cb610900 20 run: stack=0x55995f273080 is done 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb9cb610900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f30f8d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb9cb610900 4 starting trim on bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2bc2e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2bc2e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb9cb610900 20 enqueued request req=0x55995f27c6a0 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb9cb610900 20 req: 0x55995f27c6a0 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb97cff1640 20 dequeued request req=0x55995f27c6a0 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb97cff1640 20 RGWWQ: empty 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb97cff1640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 : miss 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb97cff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb9cb610900 20 run: stack=0x55995f273080 is io blocked 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb97cff1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb97cff1640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 info.flags=0x17 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb97cff1640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 to cache LRU end 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb97cff1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb97cff1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb97cff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb97cff1640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-43 : miss 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.876+0000 7fb97cff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb97cff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb97cff1640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-43 info.flags=0x0 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb97cff1640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-43 to cache LRU end 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb97cff1640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-43 : miss 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb97cff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb97cff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb97cff1640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-43 info.flags=0x0 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb97cff1640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-43 to cache LRU end 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb97cff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding source pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb97cff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding dest pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb97cff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb97cff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb97cff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2bc2e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2bc2e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2bc2e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2bc2e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f30f8d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f2bc2e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f30f8d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 20 run: stack=0x55995f273080 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f291490:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f291490:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 99fe3e25b640a15c6ff30e53cd85f50162f2fb68df33652b01a0ab8a46fc2693 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114604Z 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:99fe3e25b640a15c6ff30e53cd85f50162f2fb68df33652b01a0ab8a46fc2693 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 42389841b211719550b781fb198bddd9a6ccd382c3f87a65901ea2d235389b3a 2026-03-20T11:46:04.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=42389841b211719550b781fb198bddd9a6ccd382c3f87a65901ea2d235389b3a 2026-03-20T11:46:04.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114604Z 2026-03-20T11:46:04.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:04.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=16, curl_handle=0x55995f287050 2026-03-20T11:46:04.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb9cb610900 20 run: stack=0x55995f28ec10 is io blocked 2026-03-20T11:46:04.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.877+0000 7fb926ffd640 20 link_request req_data=0x55995f2877a0 req_data->id=16, curl_handle=0x55995f287050 2026-03-20T11:46:04.878 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:04.879+0000 7f126e9cd640 -1 req 8290687936754669415 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:04.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.885+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2dcbb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.882 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.885+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2dcbb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.882 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.885+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2dcbb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.882 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.885+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2dcbb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.882 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.885+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f27df40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.882 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.885+0000 7fb9cb610900 15 stack 0x55995f291350 end 2026-03-20T11:46:04.882 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.885+0000 7fb9cb610900 20 run: stack=0x55995f291350 is done 2026-03-20T11:46:04.882 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.885+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f271d60:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.882 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.885+0000 7fb9cb610900 20 collect(): s=0x55995f18b1a0 stack=0x55995f291350 is complete 2026-03-20T11:46:04.883 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.885+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:04.883 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.885+0000 7fb9cb610900 10 trimming bilogs for bucket=:dipepu-39[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44]) markers=[,,,,,,,,00000000001.18.6,,], shards=11 2026-03-20T11:46:04.883 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.885+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f27df40:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.883 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.885+0000 7fb9cb610900 10 trimming bilog shard 8 of :dipepu-39[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44]) at marker 00000000001.18.6 2026-03-20T11:46:04.883 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.885+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.883 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.885+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.883 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.885+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.883 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.885+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44.8 2026-03-20T11:46:04.883 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.885+0000 7fb9cb610900 20 run: stack=0x55995f291350 is io blocked 2026-03-20T11:46:04.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.886+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.886+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.886+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.886+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.886+0000 7fb9cb610900 15 stack 0x55995f291350 end 2026-03-20T11:46:04.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.886+0000 7fb9cb610900 20 run: stack=0x55995f291350 is done 2026-03-20T11:46:04.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.886+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f27df40:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.886+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f271d60:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.886+0000 7fb9cb610900 20 trim: trimmed bucket instance dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.886+0000 7fb9cb610900 15 stack 0x55995f18b1a0 end 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.886+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is done 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.886+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.886+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.886+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f271d60:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.886+0000 7fb9cb610900 4 starting trim on bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.886+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf5d80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.886+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf5d80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.886+0000 7fb9cb610900 20 enqueued request req=0x55995f283d90 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.886+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.886+0000 7fb9cb610900 20 req: 0x55995f283d90 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.886+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is io blocked 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.886+0000 7fb97c7f0640 20 dequeued request req=0x55995f283d90 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.886+0000 7fb97c7f0640 20 RGWWQ: empty 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.886+0000 7fb97c7f0640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 : miss 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.886+0000 7fb97c7f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb97c7f0640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb97c7f0640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 info.flags=0x17 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb97c7f0640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 to cache LRU end 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb97c7f0640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb97c7f0640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb97c7f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb97c7f0640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-10 : miss 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb97c7f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb97c7f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb97c7f0640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-10 info.flags=0x0 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb97c7f0640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-10 to cache LRU end 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb97c7f0640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-10 : miss 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb97c7f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb97c7f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb97c7f0640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-10 info.flags=0x0 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb97c7f0640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-10 to cache LRU end 2026-03-20T11:46:04.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb97c7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding source pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb97c7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding dest pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb97c7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb97c7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb97c7f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf5d80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf5d80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf5d80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf5d80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f271d60:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f27df40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f271d60:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2dcbb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2dcbb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 652eda1263a20c47aab91c6b6e37aab100737d0a75a27184ed0ae4ee29392cd6 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114604Z 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:652eda1263a20c47aab91c6b6e37aab100737d0a75a27184ed0ae4ee29392cd6 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 2bfe8f877025af61d8923380937fc4160059b37010adbdb59ab43de5ce4be1f7 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2bfe8f877025af61d8923380937fc4160059b37010adbdb59ab43de5ce4be1f7 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114604Z 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=17, curl_handle=0x55995f27e940 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb9cb610900 20 run: stack=0x55995f291350 is io blocked 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.887+0000 7fb926ffd640 20 link_request req_data=0x55995f27f550 req_data->id=17, curl_handle=0x55995f27e940 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2bebf0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2bebf0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2bf450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2bebf0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2bf450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2bebf0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2bf450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f287070:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 15 stack 0x55995f2947e0 end 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is done 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2bf450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 20 collect(): s=0x55995f290630 stack=0x55995f2947e0 is complete 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 10 trimming bilogs for bucket=:dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) markers=[,,,,,00000000002.22.11,,,00000000031.119.12,,], shards=11 2026-03-20T11:46:04.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2c0b60:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 15 stack 0x55995f2bfaa0 end 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 20 run: stack=0x55995f2bfaa0 is done 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f287070:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 10 trimming bilog shard 5 of :dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) at marker 00000000002.22.11 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 10 trimming bilog shard 8 of :dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) at marker 00000000031.119.12 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 20 run: stack=0x55995f290630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f28d5b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 20 collect(): s=0x55995f274bf0 stack=0x55995f2bfaa0 is complete 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 10 trimming bilogs for bucket=:dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) markers=[,00000000002.13.6,,00000000001.5.6,,00000000001.2.6,00000000002.13.6,00000000001.11.6,00000000001.4.6,00000000001.12.6,], shards=11 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2df620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f2d8be0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2bebf0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 10 trimming bilog shard 1 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000002.13.6 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 10 trimming bilog shard 3 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.5.6 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 10 trimming bilog shard 5 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.2.6 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 10 trimming bilog shard 6 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000002.13.6 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 10 trimming bilog shard 7 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.11.6 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 10 trimming bilog shard 8 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.4.6 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 10 trimming bilog shard 9 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.12.6 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 20 run: stack=0x55995f274bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2df620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21.5 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is io blocked 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f2d8be0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.888+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21.8 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 run: stack=0x55995f292d90 is io blocked 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995edfa060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f292ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f277e80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 cr:s=0x55995f289c40:op=0x55995edfa930:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995edfcc80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 cr:s=0x55995f2738e0:op=0x55995edfd480:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2bfe30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995edfa060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.1 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 run: stack=0x55995f2bfaa0 is io blocked 2026-03-20T11:46:04.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f292ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.3 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 run: stack=0x55995f293a90 is io blocked 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f277e80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.5 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 run: stack=0x55995f2be050 is io blocked 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 cr:s=0x55995f289c40:op=0x55995edfa930:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.6 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 run: stack=0x55995f289c40 is io blocked 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995edfcc80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.7 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 run: stack=0x55995f2936f0 is io blocked 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 cr:s=0x55995f2738e0:op=0x55995edfd480:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.8 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 run: stack=0x55995f2738e0 is io blocked 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2bfe30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.9 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.889+0000 7fb9cb610900 20 run: stack=0x55995f290bf0 is io blocked 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.890+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f292ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.890+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f292ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.890+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f292ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.890+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995f292ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.890+0000 7fb9cb610900 15 stack 0x55995f293a90 end 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.890+0000 7fb9cb610900 20 run: stack=0x55995f293a90 is done 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.890+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2bebf0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.890+0000 7fb9cb610900 20 run: stack=0x55995f274bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f2d8be0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f2d8be0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f2d8be0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f2d8be0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 15 stack 0x55995f292d90 end 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 run: stack=0x55995f292d90 is done 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f287070:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 run: stack=0x55995f290630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995edfa060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995edfa060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995edfa060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995edfa060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 15 stack 0x55995f2bfaa0 end 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 run: stack=0x55995f2bfaa0 is done 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2bebf0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 run: stack=0x55995f274bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995edfcc80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995edfcc80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995edfcc80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995edfcc80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 15 stack 0x55995f2936f0 end 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 run: stack=0x55995f2936f0 is done 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2bebf0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 run: stack=0x55995f274bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f2738e0:op=0x55995edfd480:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f2738e0:op=0x55995edfd480:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f2738e0:op=0x55995edfd480:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f2738e0:op=0x55995edfd480:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 15 stack 0x55995f2738e0 end 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 run: stack=0x55995f2738e0 is done 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2bebf0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 run: stack=0x55995f274bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2df620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2df620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f277e80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2bfe30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2df620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f277e80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2bfe30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2df620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 15 stack 0x55995f2947e0 end 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is done 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f277e80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2bfe30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f287070:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f277e80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 15 stack 0x55995f2be050 end 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 run: stack=0x55995f2be050 is done 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2bfe30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 15 stack 0x55995f290bf0 end 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 run: stack=0x55995f290bf0 is done 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 trim: trimmed bucket instance dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 15 stack 0x55995f290630 end 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 run: stack=0x55995f290630 is done 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2bebf0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2bebf0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 run: stack=0x55995f274bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 4 starting trim on bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 enqueued request req=0x55995f27d040 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 req: 0x55995f27d040 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 run: stack=0x55995f290630 is io blocked 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb97bfef640 20 dequeued request req=0x55995f27d040 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb97bfef640 20 RGWWQ: empty 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb97bfef640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 : miss 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f289c40:op=0x55995edfa930:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f289c40:op=0x55995edfa930:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f289c40:op=0x55995edfa930:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f289c40:op=0x55995edfa930:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 15 stack 0x55995f289c40 end 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 run: stack=0x55995f289c40 is done 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f2bebf0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f274bf0:op=0x55995f28d5b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 trim: trimmed bucket instance dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 15 stack 0x55995f274bf0 end 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 run: stack=0x55995f274bf0 is done 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2bfe30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 4 starting trim on bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:04.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb97bfef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 enqueued request req=0x55995edfb560 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 req: 0x55995edfb560 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb9cb610900 20 run: stack=0x55995f290bf0 is io blocked 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb97b7ee640 20 dequeued request req=0x55995edfb560 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb97b7ee640 20 RGWWQ: empty 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb97b7ee640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 : miss 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.891+0000 7fb97b7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97bfef640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97bfef640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 info.flags=0x17 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97bfef640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 to cache LRU end 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97bfef640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97bfef640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97bfef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97bfef640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-3 : miss 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97bfef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97bfef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97bfef640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-3 info.flags=0x0 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97bfef640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-3 to cache LRU end 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97bfef640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-3 : miss 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97bfef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97b7ee640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97b7ee640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 info.flags=0x17 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97b7ee640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 to cache LRU end 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97b7ee640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97b7ee640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97b7ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97b7ee640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-9 : miss 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97b7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97bfef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97bfef640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-3 info.flags=0x0 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97bfef640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-3 to cache LRU end 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97bfef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding source pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97bfef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding dest pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97bfef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97bfef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97bfef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:04.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.897 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:04.894+0000 7f122513a640 -1 req 7261022200790924143 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:04.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f287070:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 20 run: stack=0x55995f290630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f2bebf0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f2bebf0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:04.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:04.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.899 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.899 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.899 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.899 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:04.899 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:04.899 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:04.899 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.899 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 5cbbbf7d272a91947b385b1bca099cbe8156cb33d4287e84dbdbf93886826874 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114604Z 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:5cbbbf7d272a91947b385b1bca099cbe8156cb33d4287e84dbdbf93886826874 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 8446184e28d21f451e3e6ae5490a02c06233e9cbf3f547c3619d638ff0d65de5 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=8446184e28d21f451e3e6ae5490a02c06233e9cbf3f547c3619d638ff0d65de5 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114604Z 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=18, curl_handle=0x55995f283a50 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb9cb610900 20 run: stack=0x55995f2be050 is io blocked 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb926ffd640 20 link_request req_data=0x7fb96c0062e0 req_data->id=18, curl_handle=0x55995f283a50 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97b7ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97b7ee640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-9 info.flags=0x0 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97b7ee640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-9 to cache LRU end 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97b7ee640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-9 : miss 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.892+0000 7fb97b7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb97b7ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb97b7ee640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-9 info.flags=0x0 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb97b7ee640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-9 to cache LRU end 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb97b7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding source pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb97b7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding dest pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb97b7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb97b7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb97b7ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2bfe30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2bf450:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2bfe30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 20 run: stack=0x55995f290bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f30c8d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f30c8d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:04.901 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:04.901 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:04.901 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.901 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.901 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.901 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.901 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.901 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:04.901 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 01d343a4d82bb937524d26d44f1f59dde167a90a7f3fd747440b11793d0c3bc1 2026-03-20T11:46:04.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:04.901 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114604Z 2026-03-20T11:46:04.901 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:04.901 INFO:teuthology.orchestra.run.vm02.stderr:01d343a4d82bb937524d26d44f1f59dde167a90a7f3fd747440b11793d0c3bc1 2026-03-20T11:46:04.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:04.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:04.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:04.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:04.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 6b61e176ef785c017519f49fbbd6ddfa7f50fdf545ec665129be21393aef70ff 2026-03-20T11:46:04.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=6b61e176ef785c017519f49fbbd6ddfa7f50fdf545ec665129be21393aef70ff 2026-03-20T11:46:04.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114604Z 2026-03-20T11:46:04.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:04.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=19, curl_handle=0x55995f28b500 2026-03-20T11:46:04.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is io blocked 2026-03-20T11:46:04.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.893+0000 7fb926ffd640 20 link_request req_data=0x55995f28bc10 req_data->id=19, curl_handle=0x55995f28b500 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f291490:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f291490:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f2bc2e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb9cb610900 15 stack 0x55995f28ec10 end 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb9cb610900 20 run: stack=0x55995f28ec10 is done 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f30f8d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb9cb610900 20 collect(): s=0x55995f273080 stack=0x55995f28ec10 is complete 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb9cb610900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb9cb610900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f30f8d0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb9cb610900 15 stack 0x55995f273080 end 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb9cb610900 20 stack->operate() returned ret=-22 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb9cb610900 20 run: stack=0x55995f273080 is done 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb9cb610900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f30f8d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb9cb610900 4 starting trim on bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb9cb610900 20 enqueued request req=0x55995f27c6a0 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb9cb610900 20 req: 0x55995f27c6a0 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb9cb610900 20 run: stack=0x55995f273080 is io blocked 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb97afed640 20 dequeued request req=0x55995f27c6a0 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb97afed640 20 RGWWQ: empty 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb97afed640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 : miss 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb97afed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb97afed640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb97afed640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 info.flags=0x17 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb97afed640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 to cache LRU end 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb97afed640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:04.917 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb97afed640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb97afed640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb97afed640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-38 : miss 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.920+0000 7fb97afed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb97afed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb97afed640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-38 info.flags=0x0 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb97afed640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-38 to cache LRU end 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb97afed640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-38 : miss 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb97afed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb97afed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb97afed640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-38 info.flags=0x0 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb97afed640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-38 to cache LRU end 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb97afed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding source pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb97afed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding dest pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb97afed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb97afed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb97afed640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f30f8d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f2bc2e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f30f8d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 20 run: stack=0x55995f273080 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f291490:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f291490:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.918 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.919 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.919 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.919 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.919 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:04.919 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = bb61d736890c723ba8b48d050eebcad99bcc00658672fc1ce2a30fb1e749da10 2026-03-20T11:46:04.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:04.919 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114604Z 2026-03-20T11:46:04.919 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:04.919 INFO:teuthology.orchestra.run.vm02.stderr:bb61d736890c723ba8b48d050eebcad99bcc00658672fc1ce2a30fb1e749da10 2026-03-20T11:46:04.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:04.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:04.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:04.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:04.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 9efa5939e490c76dc8046b3fb591235ee42680d98d5be5c3bdc9b16f7c82f58e 2026-03-20T11:46:04.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=9efa5939e490c76dc8046b3fb591235ee42680d98d5be5c3bdc9b16f7c82f58e 2026-03-20T11:46:04.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114604Z 2026-03-20T11:46:04.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:04.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=20, curl_handle=0x55995f287050 2026-03-20T11:46:04.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb9cb610900 20 run: stack=0x55995f28ec10 is io blocked 2026-03-20T11:46:04.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.921+0000 7fb926ffd640 20 link_request req_data=0x55995f2877a0 req_data->id=20, curl_handle=0x55995f287050 2026-03-20T11:46:04.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.933+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2dcbb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.933+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2dcbb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.933+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2dcbb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.933+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2dcbb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.933+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f27df40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.933+0000 7fb9cb610900 15 stack 0x55995f291350 end 2026-03-20T11:46:04.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.933+0000 7fb9cb610900 20 run: stack=0x55995f291350 is done 2026-03-20T11:46:04.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.933+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f271d60:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.933+0000 7fb9cb610900 20 collect(): s=0x55995f18b1a0 stack=0x55995f291350 is complete 2026-03-20T11:46:04.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.933+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:04.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.933+0000 7fb9cb610900 10 trimming bilogs for bucket=:dipepu-10[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15]) markers=[,00000000002.9.6,,,,,,,,,], shards=11 2026-03-20T11:46:04.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.933+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2dfe50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.933+0000 7fb9cb610900 10 trimming bilog shard 1 of :dipepu-10[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15]) at marker 00000000002.9.6 2026-03-20T11:46:04.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.933+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.933+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.933+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.933+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15.1 2026-03-20T11:46:04.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.933+0000 7fb9cb610900 20 run: stack=0x55995f291350 is io blocked 2026-03-20T11:46:04.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f2bebf0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f2bebf0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f287070:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 15 stack 0x55995f2be050 end 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 run: stack=0x55995f2be050 is done 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 collect(): s=0x55995f290630 stack=0x55995f2be050 is complete 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 15 stack 0x55995f290630 end 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 stack->operate() returned ret=-22 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 run: stack=0x55995f290630 is done 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 4 starting trim on bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 enqueued request req=0x55995f27d040 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 req: 0x55995f27d040 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 run: stack=0x55995f290630 is io blocked 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb97a7ec640 20 dequeued request req=0x55995f27d040 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb97a7ec640 20 RGWWQ: empty 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb97a7ec640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 : miss 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb97a7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 15 stack 0x55995f291350 end 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 run: stack=0x55995f291350 is done 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2dfe50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f271d60:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 trim: trimmed bucket instance dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 15 stack 0x55995f18b1a0 end 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb97a7ec640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is done 2026-03-20T11:46:04.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:04.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb97a7ec640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 info.flags=0x17 2026-03-20T11:46:04.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb97a7ec640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 to cache LRU end 2026-03-20T11:46:04.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb97a7ec640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:04.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb97a7ec640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:04.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb97a7ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:04.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb97a7ec640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-15 : miss 2026-03-20T11:46:04.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb97a7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edfc9f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 4 starting trim on bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:04.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f28d4d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f28d4d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 enqueued request req=0x55995f283d90 2026-03-20T11:46:04.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 req: 0x55995f283d90 2026-03-20T11:46:04.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is io blocked 2026-03-20T11:46:04.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb979feb640 20 dequeued request req=0x55995f283d90 2026-03-20T11:46:04.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb979feb640 20 RGWWQ: empty 2026-03-20T11:46:04.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb979feb640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 : miss 2026-03-20T11:46:04.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.935+0000 7fb979feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb97a7ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb97a7ec640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-15 info.flags=0x0 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb97a7ec640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-15 to cache LRU end 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb979feb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb97a7ec640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-15 : miss 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb979feb640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 info.flags=0x17 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb97a7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb979feb640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 to cache LRU end 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb979feb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb979feb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb979feb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb979feb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-13 : miss 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb979feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb97a7ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb97a7ec640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-15 info.flags=0x0 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb97a7ec640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-15 to cache LRU end 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb979feb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb979feb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-13 info.flags=0x0 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb979feb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-13 to cache LRU end 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb979feb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-13 : miss 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb97a7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding source pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb979feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb97a7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding dest pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb97a7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb97a7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb97a7ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f279750:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb9cb610900 20 run: stack=0x55995f290630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2dcbb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2dcbb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:04.934 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 920368bb25e04df4ae8f43ca14bf1131ff84bfda33bde2734dea87b19fbaac48 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114604Z 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:920368bb25e04df4ae8f43ca14bf1131ff84bfda33bde2734dea87b19fbaac48 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb979feb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb979feb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-13 info.flags=0x0 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb979feb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-13 to cache LRU end 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb979feb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18): adding source pipe: {s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb979feb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18): adding dest pipe: {s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb979feb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb979feb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb979feb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = b4f0f6cd8bc22340925e21c2cc6c077be81a749fde6de0ecc858addf03a8bfa9 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b4f0f6cd8bc22340925e21c2cc6c077be81a749fde6de0ecc858addf03a8bfa9 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.936+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114604Z 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=21, curl_handle=0x55995f283a50 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 20 run: stack=0x55995f291350 is io blocked 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f28d4d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb926ffd640 20 link_request req_data=0x7fb96c0062e0 req_data->id=21, curl_handle=0x55995f283a50 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f30c8d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f28d4d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f30c8d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f28d4d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f30c8d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f28d4d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f30c8d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edfc9f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2bf450:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 15 stack 0x55995f2947e0 end 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is done 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f30c8d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edfc9f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2bfe30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 20 collect(): s=0x55995f290bf0 stack=0x55995f2947e0 is complete 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 10 trimming bilogs for bucket=:dipepu-9[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14]) markers=[,00000000002.17.6,,,,,,,,,], shards=11 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f2bf450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2bebf0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 10 trimming bilog shard 1 of :dipepu-9[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14]) at marker 00000000002.17.6 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 20 run: stack=0x55995f290bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f2bf450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.937+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.938+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.938+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.938+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.938+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.938+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.938+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 91df7049cf51250681eb5cc62dfbdeaf8921f0fb52b557705807409b4d6c5c3b 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.938+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114604Z 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:91df7049cf51250681eb5cc62dfbdeaf8921f0fb52b557705807409b4d6c5c3b 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.938+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.938+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.938+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.938+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.938+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 5602cd32372272f9fd271a193e30254cd726786fda2b14cdf666d46961788363 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.938+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5602cd32372272f9fd271a193e30254cd726786fda2b14cdf666d46961788363 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.938+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.938+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114604Z 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.938+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:04.936 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.938+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=22, curl_handle=0x55995f28b500 2026-03-20T11:46:04.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.938+0000 7fb926ffd640 20 link_request req_data=0x55995f28bc10 req_data->id=22, curl_handle=0x55995f28b500 2026-03-20T11:46:04.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.938+0000 7fb9cb610900 20 run: stack=0x55995f2be050 is io blocked 2026-03-20T11:46:04.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.938+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f28ed50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.938+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f28ed50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.938+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14.1 2026-03-20T11:46:04.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.938+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is io blocked 2026-03-20T11:46:04.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f28ed50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f28ed50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f28ed50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f28ed50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9cb610900 15 stack 0x55995f2947e0 end 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is done 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2bebf0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2bfe30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9cb610900 20 trim: trimmed bucket instance dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9cb610900 15 stack 0x55995f290bf0 end 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9cb610900 20 run: stack=0x55995f290bf0 is done 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2bfe30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9cb610900 4 starting trim on bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2d8f90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2d8f90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9cb610900 20 enqueued request req=0x55995f283d90 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9cb610900 20 req: 0x55995f283d90 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9cb610900 20 run: stack=0x55995f290bf0 is io blocked 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9797ea640 20 dequeued request req=0x55995f283d90 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9797ea640 20 RGWWQ: empty 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9797ea640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 : miss 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9797ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9797ea640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9797ea640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 info.flags=0x17 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9797ea640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 to cache LRU end 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9797ea640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9797ea640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9797ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9797ea640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-11 : miss 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9797ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9797ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9797ea640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-11 info.flags=0x0 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9797ea640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-11 to cache LRU end 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9797ea640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-11 : miss 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.941+0000 7fb9797ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9797ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9797ea640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-11 info.flags=0x0 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9797ea640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-11 to cache LRU end 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9797ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16): adding source pipe: {s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9797ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16): adding dest pipe: {s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9797ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9797ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9797ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2d8f90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2d8f90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2d8f90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2d8f90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2bfe30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f287070:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2bfe30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 20 run: stack=0x55995f290bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2dbf80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2dbf80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = bd23fc0da95b59437048ff0e06c5b4c9a21b63de723c2e0294ada3ff03f9323d 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114604Z 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:bd23fc0da95b59437048ff0e06c5b4c9a21b63de723c2e0294ada3ff03f9323d 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 15f08f4b7444f40fa964ce0b26adf812bc35403e735c8e966c93d3f1738cbafc 2026-03-20T11:46:04.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=15f08f4b7444f40fa964ce0b26adf812bc35403e735c8e966c93d3f1738cbafc 2026-03-20T11:46:04.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114604Z 2026-03-20T11:46:04.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:04.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=23, curl_handle=0x55995f27e940 2026-03-20T11:46:04.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is io blocked 2026-03-20T11:46:04.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.942+0000 7fb926ffd640 20 link_request req_data=0x55995f27f550 req_data->id=23, curl_handle=0x55995f27e940 2026-03-20T11:46:04.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.965+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f291490:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.965+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f291490:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.965+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f291490:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.965+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f291490:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.965+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f2bc2e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.965+0000 7fb9cb610900 15 stack 0x55995f28ec10 end 2026-03-20T11:46:04.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.965+0000 7fb9cb610900 20 run: stack=0x55995f28ec10 is done 2026-03-20T11:46:04.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.965+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f30f8d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.965+0000 7fb9cb610900 20 collect(): s=0x55995f273080 stack=0x55995f28ec10 is complete 2026-03-20T11:46:04.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.965+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:04.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.965+0000 7fb9cb610900 10 trimming bilogs for bucket=:dipepu-38[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43]) markers=[,,,,,,,,00000000001.55.6,,], shards=11 2026-03-20T11:46:04.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.965+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2bebf0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.965+0000 7fb9cb610900 10 trimming bilog shard 8 of :dipepu-38[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43]) at marker 00000000001.55.6 2026-03-20T11:46:04.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.965+0000 7fb9cb610900 20 run: stack=0x55995f273080 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.962 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.965+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f292ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.963 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.965+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f292ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.963 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.965+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43.8 2026-03-20T11:46:04.963 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.965+0000 7fb9cb610900 20 run: stack=0x55995f28ec10 is io blocked 2026-03-20T11:46:04.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f292ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f292ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f292ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f292ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb9cb610900 15 stack 0x55995f28ec10 end 2026-03-20T11:46:04.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb9cb610900 20 run: stack=0x55995f28ec10 is done 2026-03-20T11:46:04.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2bebf0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f30f8d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb9cb610900 20 trim: trimmed bucket instance dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:04.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb9cb610900 15 stack 0x55995f273080 end 2026-03-20T11:46:04.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb9cb610900 20 run: stack=0x55995f273080 is done 2026-03-20T11:46:04.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:04.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f277680:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb9cb610900 4 starting trim on bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:04.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2d8f90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2d8f90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb9cb610900 20 enqueued request req=0x55995f27c6a0 2026-03-20T11:46:04.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb9cb610900 20 req: 0x55995f27c6a0 2026-03-20T11:46:04.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb9cb610900 20 run: stack=0x55995f273080 is io blocked 2026-03-20T11:46:04.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb978fe9640 20 dequeued request req=0x55995f27c6a0 2026-03-20T11:46:04.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb978fe9640 20 RGWWQ: empty 2026-03-20T11:46:04.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb978fe9640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 : miss 2026-03-20T11:46:04.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb978fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb978fe9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:04.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb978fe9640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 info.flags=0x17 2026-03-20T11:46:04.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb978fe9640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 to cache LRU end 2026-03-20T11:46:04.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb978fe9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:04.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb978fe9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:04.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb978fe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:04.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb978fe9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-12 : miss 2026-03-20T11:46:04.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.967+0000 7fb978fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb978fe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb978fe9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-12 info.flags=0x0 2026-03-20T11:46:04.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb978fe9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-12 to cache LRU end 2026-03-20T11:46:04.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb978fe9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-12 : miss 2026-03-20T11:46:04.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb978fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb978fe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb978fe9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-12 info.flags=0x0 2026-03-20T11:46:04.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb978fe9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-12 to cache LRU end 2026-03-20T11:46:04.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb978fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17): adding source pipe: {s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb978fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17): adding dest pipe: {s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb978fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb978fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb978fe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2d8f90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2d8f90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2d8f90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2d8f90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f277680:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f2bebc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f277680:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 20 run: stack=0x55995f273080 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f291490:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f291490:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 7bb0934329545dce8403f89bc470a1f8649e68270b301e669d1baef3704dc71b 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114604Z 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:7bb0934329545dce8403f89bc470a1f8649e68270b301e669d1baef3704dc71b 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 2dfbccdd1c03ee3e5cf292e395938da7ca51d3cd807d17011e46dc26d8f250ce 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2dfbccdd1c03ee3e5cf292e395938da7ca51d3cd807d17011e46dc26d8f250ce 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114604Z 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=24, curl_handle=0x55995f287050 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb9cb610900 20 run: stack=0x55995f28ec10 is io blocked 2026-03-20T11:46:04.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.968+0000 7fb926ffd640 20 link_request req_data=0x55995f2877a0 req_data->id=24, curl_handle=0x55995f287050 2026-03-20T11:46:04.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2dcbb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2dcbb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f2bf450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2dcbb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f2bf450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2dcbb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f2bf450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f279750:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 15 stack 0x55995f291350 end 2026-03-20T11:46:04.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 run: stack=0x55995f291350 is done 2026-03-20T11:46:04.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f2bf450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 collect(): s=0x55995f290630 stack=0x55995f291350 is complete 2026-03-20T11:46:04.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:04.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 10 trimming bilogs for bucket=:dipepu-15[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20]) markers=[,,,,,,,,00000000007.15.6,,], shards=11 2026-03-20T11:46:04.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f30c8d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 15 stack 0x55995f2be050 end 2026-03-20T11:46:04.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 run: stack=0x55995f2be050 is done 2026-03-20T11:46:04.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f27df40:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 10 trimming bilog shard 8 of :dipepu-15[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20]) at marker 00000000007.15.6 2026-03-20T11:46:04.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 run: stack=0x55995f290630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edfc9f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 collect(): s=0x55995f18b1a0 stack=0x55995f2be050 is complete 2026-03-20T11:46:04.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:04.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 10 trimming bilogs for bucket=:dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) markers=[,,,,,00000000002.36.11,,,00000000011.25.6,00000000003.20.6,], shards=11 2026-03-20T11:46:04.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bc2e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 10 trimming bilog shard 5 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000002.36.11 2026-03-20T11:46:04.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 10 trimming bilog shard 8 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000011.25.6 2026-03-20T11:46:04.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 10 trimming bilog shard 9 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000003.20.6 2026-03-20T11:46:04.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20.8 2026-03-20T11:46:04.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 run: stack=0x55995f291350 is io blocked 2026-03-20T11:46:04.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f276e10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2d84a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.5 2026-03-20T11:46:04.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 run: stack=0x55995f2be050 is io blocked 2026-03-20T11:46:04.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f276e10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.8 2026-03-20T11:46:04.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 run: stack=0x55995f2936f0 is io blocked 2026-03-20T11:46:04.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2d84a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.9 2026-03-20T11:46:04.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.982+0000 7fb9cb610900 20 run: stack=0x55995f2bfaa0 is io blocked 2026-03-20T11:46:04.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2dbf80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2dbf80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2d84a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f276e10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2dbf80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2d84a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f276e10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2dbf80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2d84a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f276e10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f287070:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 15 stack 0x55995f2947e0 end 2026-03-20T11:46:04.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is done 2026-03-20T11:46:04.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2d84a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 15 stack 0x55995f2bfaa0 end 2026-03-20T11:46:04.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 run: stack=0x55995f2bfaa0 is done 2026-03-20T11:46:04.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f276e10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 15 stack 0x55995f2936f0 end 2026-03-20T11:46:04.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 run: stack=0x55995f2936f0 is done 2026-03-20T11:46:04.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 15 stack 0x55995f291350 end 2026-03-20T11:46:04.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 run: stack=0x55995f291350 is done 2026-03-20T11:46:04.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2bfe30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 collect(): s=0x55995f290bf0 stack=0x55995f2947e0 is complete 2026-03-20T11:46:04.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:04.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 10 trimming bilogs for bucket=:dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) markers=[00000000014.53.6,00000000008.38.6,00000000008.39.6,00000000008.35.6,00000000012.28.6,00000000010.38.6,00000000008.22.6,00000000010.27.6,00000000004.28.6,00000000008.20.6,00000000008.23.6], shards=11 2026-03-20T11:46:04.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bc2e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f27df40:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f279750:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 10 trimming bilog shard 0 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000014.53.6 2026-03-20T11:46:04.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 10 trimming bilog shard 1 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.38.6 2026-03-20T11:46:04.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 10 trimming bilog shard 2 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.39.6 2026-03-20T11:46:04.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 10 trimming bilog shard 3 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.35.6 2026-03-20T11:46:04.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 10 trimming bilog shard 4 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000012.28.6 2026-03-20T11:46:04.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 10 trimming bilog shard 5 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000010.38.6 2026-03-20T11:46:04.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 10 trimming bilog shard 6 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.22.6 2026-03-20T11:46:04.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 10 trimming bilog shard 7 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000010.27.6 2026-03-20T11:46:04.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 10 trimming bilog shard 8 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000004.28.6 2026-03-20T11:46:04.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 10 trimming bilog shard 9 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.20.6 2026-03-20T11:46:04.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 10 trimming bilog shard 10 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.23.6 2026-03-20T11:46:04.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 run: stack=0x55995f290bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bc2e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 trim: trimmed bucket instance dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:04.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 15 stack 0x55995f290630 end 2026-03-20T11:46:04.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 run: stack=0x55995f290630 is done 2026-03-20T11:46:04.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995edf7e60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2d84a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f293bd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995edfa400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f28d440:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f2c6310:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995edfc270:op=0x55995f271d60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f310640:op=0x55995f2df8e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f2c6b10:op=0x55995f276be0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f2bdc30:op=0x55995f2816e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.0 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 run: stack=0x55995f291350 is io blocked 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995edf7e60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.1 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 run: stack=0x55995f2936f0 is io blocked 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2d84a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.2 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is io blocked 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f293bd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.3 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 run: stack=0x55995f292d90 is io blocked 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995edfa400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.4 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 run: stack=0x55995f293a90 is io blocked 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f28d440:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.5 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 run: stack=0x55995f272de0 is io blocked 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f2c6310:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.6 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 run: stack=0x55995f28dc40 is io blocked 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995edfc270:op=0x55995f271d60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.7 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 run: stack=0x55995edfc270 is io blocked 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f310640:op=0x55995f2df8e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.8 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 run: stack=0x55995f310640 is io blocked 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f2c6b10:op=0x55995f276be0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.9 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 run: stack=0x55995f2c6b10 is io blocked 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 cr:s=0x55995f2bdc30:op=0x55995f2816e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.985+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.10 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 20 run: stack=0x55995f2bdc30 is io blocked 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 4 starting trim on bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995edf59b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995edf59b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 20 enqueued request req=0x55995f282190 2026-03-20T11:46:04.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 20 req: 0x55995f282190 2026-03-20T11:46:04.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 20 run: stack=0x55995f290630 is io blocked 2026-03-20T11:46:04.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 15 stack 0x55995f2be050 end 2026-03-20T11:46:04.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 20 run: stack=0x55995f2be050 is done 2026-03-20T11:46:04.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bc2e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edfc9f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 20 trim: trimmed bucket instance dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:04.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 15 stack 0x55995f18b1a0 end 2026-03-20T11:46:04.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is done 2026-03-20T11:46:04.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:04.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edfc9c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 4 starting trim on bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:04.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 20 enqueued request req=0x55995f27d040 2026-03-20T11:46:04.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 20 req: 0x55995f282190 2026-03-20T11:46:04.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 20 req: 0x55995f27d040 2026-03-20T11:46:04.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.987+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is io blocked 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.988+0000 7fb9787e8640 20 dequeued request req=0x55995f282190 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.988+0000 7fb9787e8640 20 RGWWQ: 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.988+0000 7fb9787e8640 20 req: 0x55995f27d040 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.988+0000 7fb9787e8640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 : miss 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.988+0000 7fb9787e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.988+0000 7fb977fe7640 20 dequeued request req=0x55995f27d040 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.988+0000 7fb977fe7640 20 RGWWQ: empty 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.988+0000 7fb977fe7640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 : miss 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.988+0000 7fb977fe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.988+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f28d440:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9787e8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9787e8640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 info.flags=0x17 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9787e8640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 to cache LRU end 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9787e8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9787e8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9787e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f28d440:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995edfc270:op=0x55995f271d60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f2c6310:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2d84a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f28d440:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995edfc270:op=0x55995f271d60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f2c6310:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2d84a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f28d440:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 15 stack 0x55995f272de0 end 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 run: stack=0x55995f272de0 is done 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995edfc270:op=0x55995f271d60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f2c6310:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2d84a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f279750:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 run: stack=0x55995f290bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995edfc270:op=0x55995f271d60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 15 stack 0x55995edfc270 end 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 run: stack=0x55995edfc270 is done 2026-03-20T11:46:04.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 15 stack 0x55995f291350 end 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 run: stack=0x55995f291350 is done 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f2c6310:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 15 stack 0x55995f28dc40 end 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 run: stack=0x55995f28dc40 is done 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2d84a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 15 stack 0x55995f2947e0 end 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is done 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f279750:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f279750:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f279750:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f279750:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 run: stack=0x55995f290bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995edf7e60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995edf7e60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995edf7e60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995edf7e60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 15 stack 0x55995f2936f0 end 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 run: stack=0x55995f2936f0 is done 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f279750:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 run: stack=0x55995f290bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9787e8640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-40 : miss 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9787e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f293bd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f293bd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f293bd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f292d90:op=0x55995f293bd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 15 stack 0x55995f292d90 end 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 run: stack=0x55995f292d90 is done 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f279750:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 run: stack=0x55995f290bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995edfa400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995edfa400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f310640:op=0x55995f2df8e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995edfa400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f310640:op=0x55995f2df8e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f293a90:op=0x55995edfa400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 15 stack 0x55995f293a90 end 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 run: stack=0x55995f293a90 is done 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f310640:op=0x55995f2df8e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f279750:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 run: stack=0x55995f290bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f310640:op=0x55995f2df8e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 15 stack 0x55995f310640 end 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 run: stack=0x55995f310640 is done 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f279750:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9cb610900 20 run: stack=0x55995f290bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb977fe7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb977fe7640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 info.flags=0x17 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb977fe7640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 to cache LRU end 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb977fe7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb977fe7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb977fe7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb977fe7640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-18 : miss 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb977fe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9787e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9787e8640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-40 info.flags=0x0 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9787e8640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-40 to cache LRU end 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9787e8640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-40 : miss 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.989+0000 7fb9787e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f2c6b10:op=0x55995f276be0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f2c6b10:op=0x55995f276be0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f2c6b10:op=0x55995f276be0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f2c6b10:op=0x55995f276be0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 15 stack 0x55995f2c6b10 end 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 run: stack=0x55995f2c6b10 is done 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f279750:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 run: stack=0x55995f290bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb977fe7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb977fe7640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-18 info.flags=0x0 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb977fe7640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-18 to cache LRU end 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb977fe7640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-18 : miss 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb977fe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb977fe7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb977fe7640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-18 info.flags=0x0 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb977fe7640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-18 to cache LRU end 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb977fe7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23): adding source pipe: {s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f2bdc30:op=0x55995f2816e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f2bdc30:op=0x55995f2816e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f2bdc30:op=0x55995f2816e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f2bdc30:op=0x55995f2816e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 15 stack 0x55995f2bdc30 end 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 run: stack=0x55995f2bdc30 is done 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f279750:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f290bf0:op=0x55995f2bfe30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 trim: trimmed bucket instance dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 15 stack 0x55995f290bf0 end 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 run: stack=0x55995f290bf0 is done 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb977fe7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23): adding dest pipe: {s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f2bfbe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 4 starting trim on bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f271d60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f271d60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 enqueued request req=0x55995f275400 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 req: 0x55995f275400 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 run: stack=0x55995f2936f0 is io blocked 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9777e6640 20 dequeued request req=0x55995f275400 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9777e6640 20 RGWWQ: empty 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9777e6640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 : miss 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9777e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb977fe7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb977fe7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb977fe7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9787e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edfc9c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9787e8640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-40 info.flags=0x0 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9787e8640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-40 to cache LRU end 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9787e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1): adding source pipe: {s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f2bc2e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edfc9c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995edf7790:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9787e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1): adding dest pipe: {s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995edf7790:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9787e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9787e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9787e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 261e9abd81567687cea6f617086ac3a4abd1661d064082e721721405bbeea011 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114604Z 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:261e9abd81567687cea6f617086ac3a4abd1661d064082e721721405bbeea011 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 575b77ebadfb3fdf4efb6d7cec79563d1086a097f6cde32b1d8e627cb4acaf8a 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=575b77ebadfb3fdf4efb6d7cec79563d1086a097f6cde32b1d8e627cb4acaf8a 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114604Z 2026-03-20T11:46:04.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:04.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=25, curl_handle=0x55995f27e940 2026-03-20T11:46:04.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 run: stack=0x55995f28dc40 is io blocked 2026-03-20T11:46:04.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995edf59b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995edf59b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995edf59b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995edf59b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2dbf80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 run: stack=0x55995f290630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2763b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2763b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb926ffd640 20 link_request req_data=0x55995f27f550 req_data->id=25, curl_handle=0x55995f27e940 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 850254d1c2dafa2c3e0e94913f48d6329c01575d58688d57d48954c25934582e 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114604Z 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:850254d1c2dafa2c3e0e94913f48d6329c01575d58688d57d48954c25934582e 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 1eb862d9a0ef606044684d99c148cc3023003819f66bc68b8f53a4990493a778 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=1eb862d9a0ef606044684d99c148cc3023003819f66bc68b8f53a4990493a778 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114604Z 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9777e6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=26, curl_handle=0x55995f28b500 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9777e6640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 info.flags=0x17 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is io blocked 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9777e6640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 to cache LRU end 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9777e6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb9777e6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.990+0000 7fb926ffd640 20 link_request req_data=0x55995f28bc10 req_data->id=26, curl_handle=0x55995f28b500 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.991+0000 7fb9777e6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.991+0000 7fb9777e6640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-24 : miss 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.991+0000 7fb9777e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.991+0000 7fb9777e6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.991+0000 7fb9777e6640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-24 info.flags=0x0 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.991+0000 7fb9777e6640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-24 to cache LRU end 2026-03-20T11:46:04.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.991+0000 7fb9777e6640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-24 : miss 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.991+0000 7fb9777e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9777e6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9777e6640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-24 info.flags=0x0 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9777e6640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-24 to cache LRU end 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9777e6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29): adding source pipe: {s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9777e6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29): adding dest pipe: {s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9777e6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9777e6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9777e6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f271d60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f271d60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f271d60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f271d60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f2bfbe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f287070:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f2bfbe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 20 run: stack=0x55995f2936f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995edfdc90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995edfdc90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:04 +0000 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114604Z 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 1fa34715d3d9ee5eb5ab37dcf1b15c57ca88e4afc2b02567795945b7279dc3a3 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114604Z 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:1fa34715d3d9ee5eb5ab37dcf1b15c57ca88e4afc2b02567795945b7279dc3a3 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 2176ff82f2453537beca543c868af85d88d99bfc22926cefc58f33ddec3e68be 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2176ff82f2453537beca543c868af85d88d99bfc22926cefc58f33ddec3e68be 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114604Z 2026-03-20T11:46:04.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:04.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=27, curl_handle=0x55995f283a50 2026-03-20T11:46:04.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb9cb610900 20 run: stack=0x55995f291350 is io blocked 2026-03-20T11:46:04.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:04.992+0000 7fb926ffd640 20 link_request req_data=0x7fb96c0062e0 req_data->id=27, curl_handle=0x55995f283a50 2026-03-20T11:46:04.995 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:04.994+0000 7f12a4238640 -1 req 8125091009250537222 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:05.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f291490:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f291490:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f291490:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f291490:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f2bebc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 15 stack 0x55995f28ec10 end 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 run: stack=0x55995f28ec10 is done 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f277680:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 collect(): s=0x55995f273080 stack=0x55995f28ec10 is complete 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 10 trimming bilogs for bucket=:dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) markers=[00000000014.49.6,00000000008.48.6,00000000008.43.6,00000000008.23.6,00000000012.30.6,00000000010.25.6,00000000008.36.6,00000000010.29.6,00000000004.42.6,00000000008.28.6,00000000008.26.6], shards=11 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2bebc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 10 trimming bilog shard 0 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000014.49.6 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 10 trimming bilog shard 1 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.48.6 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 10 trimming bilog shard 2 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.43.6 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 10 trimming bilog shard 3 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.23.6 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 10 trimming bilog shard 4 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000012.30.6 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 10 trimming bilog shard 5 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000010.25.6 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 10 trimming bilog shard 6 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.36.6 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 10 trimming bilog shard 7 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000010.29.6 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 10 trimming bilog shard 8 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000004.42.6 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 10 trimming bilog shard 9 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.28.6 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 10 trimming bilog shard 10 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.26.6 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 run: stack=0x55995f273080 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f2c6970:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995edf8210:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f285b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f2e48b0:op=0x55995edfa400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2816e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f272f20:op=0x55995f293af0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f2c6680:op=0x55995f2c7320:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f2dfb50:op=0x55995edf9390:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f2c4df0:op=0x55995edf66f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.0 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 run: stack=0x55995f28ec10 is io blocked 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.1 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 run: stack=0x55995f272de0 is io blocked 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f2c6970:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.2 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 run: stack=0x55995f2be050 is io blocked 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995edf8210:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.3 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 run: stack=0x55995f2bfaa0 is io blocked 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f285b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.4 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 run: stack=0x55995f30f990 is io blocked 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f2e48b0:op=0x55995edfa400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.5 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 run: stack=0x55995f2e48b0 is io blocked 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2816e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.6 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 run: stack=0x55995f276250 is io blocked 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f272f20:op=0x55995f293af0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.7 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 run: stack=0x55995f272f20 is io blocked 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f2c6680:op=0x55995f2c7320:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.8 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 run: stack=0x55995f2c6680 is io blocked 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f2dfb50:op=0x55995edf9390:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.9 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 run: stack=0x55995f2dfb50 is io blocked 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 cr:s=0x55995f2c4df0:op=0x55995edf66f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.10 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.012+0000 7fb9cb610900 20 run: stack=0x55995f2c4df0 is io blocked 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.014+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f285b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.014+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f285b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.014+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f285b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.014+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f285b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.014+0000 7fb9cb610900 15 stack 0x55995f30f990 end 2026-03-20T11:46:05.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.014+0000 7fb9cb610900 20 run: stack=0x55995f30f990 is done 2026-03-20T11:46:05.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.014+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2bebc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.014+0000 7fb9cb610900 20 run: stack=0x55995f273080 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.014+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.014+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.014+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.014+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.014+0000 7fb9cb610900 15 stack 0x55995f28ec10 end 2026-03-20T11:46:05.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.014+0000 7fb9cb610900 20 run: stack=0x55995f28ec10 is done 2026-03-20T11:46:05.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.014+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2bebc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.014+0000 7fb9cb610900 20 run: stack=0x55995f273080 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f2c6680:op=0x55995f2c7320:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f2c6680:op=0x55995f2c7320:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f2e48b0:op=0x55995edfa400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995edf8210:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2816e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f272f20:op=0x55995f293af0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f2c6970:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f2c6680:op=0x55995f2c7320:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f2e48b0:op=0x55995edfa400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995edf8210:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2816e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f272f20:op=0x55995f293af0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f2c4df0:op=0x55995edf66f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f2c6970:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f2c6680:op=0x55995f2c7320:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 15 stack 0x55995f2c6680 end 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 run: stack=0x55995f2c6680 is done 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f2e48b0:op=0x55995edfa400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995edf8210:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2816e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f272f20:op=0x55995f293af0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f2c4df0:op=0x55995edf66f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f2c6970:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2bebc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 run: stack=0x55995f273080 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f2e48b0:op=0x55995edfa400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 15 stack 0x55995f2e48b0 end 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 run: stack=0x55995f2e48b0 is done 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995edf8210:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 15 stack 0x55995f2bfaa0 end 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 run: stack=0x55995f2bfaa0 is done 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2816e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 15 stack 0x55995f276250 end 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 run: stack=0x55995f276250 is done 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f272f20:op=0x55995f293af0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 15 stack 0x55995f272f20 end 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 run: stack=0x55995f272f20 is done 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 15 stack 0x55995f272de0 end 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 run: stack=0x55995f272de0 is done 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f2c4df0:op=0x55995edf66f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f2c6970:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 15 stack 0x55995f2be050 end 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 run: stack=0x55995f2be050 is done 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2bebc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f2c4df0:op=0x55995edf66f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 15 stack 0x55995f2c4df0 end 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 run: stack=0x55995f2c4df0 is done 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2bebc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2bebc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2bebc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2bebc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2bebc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2bebc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 run: stack=0x55995f273080 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f2dfb50:op=0x55995edf9390:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f2dfb50:op=0x55995edf9390:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f2dfb50:op=0x55995edf9390:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f2dfb50:op=0x55995edf9390:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 15 stack 0x55995f2dfb50 end 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 run: stack=0x55995f2dfb50 is done 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f2bebc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.015+0000 7fb9cb610900 20 cr:s=0x55995f273080:op=0x55995f277680:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb9cb610900 20 trim: trimmed bucket instance dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:05.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb9cb610900 15 stack 0x55995f273080 end 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb9cb610900 20 run: stack=0x55995f273080 is done 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f277680:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb9cb610900 4 starting trim on bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb9cb610900 20 enqueued request req=0x55995f2dcbb0 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb9cb610900 20 req: 0x55995f2dcbb0 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb9cb610900 20 run: stack=0x55995f276250 is io blocked 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb976fe5640 20 dequeued request req=0x55995f2dcbb0 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb976fe5640 20 RGWWQ: empty 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb976fe5640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 : miss 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb976fe5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb976fe5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb976fe5640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 info.flags=0x17 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb976fe5640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 to cache LRU end 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb976fe5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb976fe5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb976fe5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb976fe5640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-56 : miss 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb976fe5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb976fe5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb976fe5640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-56 info.flags=0x0 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb976fe5640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-56 to cache LRU end 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb976fe5640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-56 : miss 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.016+0000 7fb976fe5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb976fe5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb976fe5640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-56 info.flags=0x0 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb976fe5640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-56 to cache LRU end 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb976fe5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16): adding source pipe: {s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb976fe5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16): adding dest pipe: {s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb976fe5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb976fe5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb976fe5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2e1550:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f277680:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2bebc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f277680:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 20 run: stack=0x55995f276250 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2e1550:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2e1550:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 41b99c7755f302cda920ffc2dac1784a643713214109bbd349669e70356ae78f 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114605Z 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:41b99c7755f302cda920ffc2dac1784a643713214109bbd349669e70356ae78f 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:05.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = d0f3be2022142f78e6b4ee8ee0d7c56660137428aee98fa5626b160c0a0f13d9 2026-03-20T11:46:05.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=d0f3be2022142f78e6b4ee8ee0d7c56660137428aee98fa5626b160c0a0f13d9 2026-03-20T11:46:05.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114605Z 2026-03-20T11:46:05.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:05.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=28, curl_handle=0x55995f287050 2026-03-20T11:46:05.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb9cb610900 20 run: stack=0x55995f2bfaa0 is io blocked 2026-03-20T11:46:05.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.017+0000 7fb926ffd640 20 link_request req_data=0x55995f2877a0 req_data->id=28, curl_handle=0x55995f287050 2026-03-20T11:46:05.016 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:05.019+0000 7f12dc2a8640 -1 req 15161640487424284191 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:05.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995edfdc90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995edfdc90:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:05.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f287070:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb9cb610900 15 stack 0x55995f291350 end 2026-03-20T11:46:05.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb9cb610900 20 run: stack=0x55995f291350 is done 2026-03-20T11:46:05.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f2bfbe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb9cb610900 20 collect(): s=0x55995f2936f0 stack=0x55995f291350 is complete 2026-03-20T11:46:05.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:05.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb9cb610900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:05.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb9cb610900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:05.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f2bfbe0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:05.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb9cb610900 15 stack 0x55995f2936f0 end 2026-03-20T11:46:05.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb9cb610900 20 stack->operate() returned ret=-22 2026-03-20T11:46:05.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb9cb610900 20 run: stack=0x55995f2936f0 is done 2026-03-20T11:46:05.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:05.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb9cb610900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:05.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f2bfbe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb9cb610900 4 starting trim on bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:46:05.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995edf59b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995edf59b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb9cb610900 20 enqueued request req=0x55995f275400 2026-03-20T11:46:05.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:05.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb9cb610900 20 req: 0x55995f275400 2026-03-20T11:46:05.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb9cb610900 20 run: stack=0x55995f2936f0 is io blocked 2026-03-20T11:46:05.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb98effd640 20 dequeued request req=0x55995f275400 2026-03-20T11:46:05.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb98effd640 20 RGWWQ: empty 2026-03-20T11:46:05.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb98effd640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 : miss 2026-03-20T11:46:05.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.035+0000 7fb98effd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb98effd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:05.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb98effd640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 info.flags=0x17 2026-03-20T11:46:05.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb98effd640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 to cache LRU end 2026-03-20T11:46:05.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb98effd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:05.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb98effd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:05.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb98effd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:46:05.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb98effd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-27 : miss 2026-03-20T11:46:05.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb98effd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995edf7790:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb98effd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb98effd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-27 info.flags=0x0 2026-03-20T11:46:05.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb98effd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-27 to cache LRU end 2026-03-20T11:46:05.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb98effd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-27 : miss 2026-03-20T11:46:05.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb98effd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995edf7790:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995edf7790:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995edf7790:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f2bc2e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb9cb610900 15 stack 0x55995f28dc40 end 2026-03-20T11:46:05.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb9cb610900 20 run: stack=0x55995f28dc40 is done 2026-03-20T11:46:05.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edfc9c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb9cb610900 20 collect(): s=0x55995f18b1a0 stack=0x55995f28dc40 is complete 2026-03-20T11:46:05.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:05.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb9cb610900 10 trimming bilogs for bucket=:dipepu-18[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23]) markers=[,,,,,,,,00000000002.59.6,,], shards=11 2026-03-20T11:46:05.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bc2e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb9cb610900 10 trimming bilog shard 8 of :dipepu-18[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23]) at marker 00000000002.59.6 2026-03-20T11:46:05.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23.8 2026-03-20T11:46:05.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb9cb610900 20 run: stack=0x55995f28dc40 is io blocked 2026-03-20T11:46:05.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb98effd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb98effd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-27 info.flags=0x0 2026-03-20T11:46:05.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb98effd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-27 to cache LRU end 2026-03-20T11:46:05.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb98effd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32): adding source pipe: {s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb98effd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32): adding dest pipe: {s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb98effd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb98effd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb98effd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:46:05.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995edf59b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995edf59b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995edf59b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995edf59b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f2bfbe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.036+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2dfab0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.037+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f2bfbe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.037+0000 7fb9cb610900 20 run: stack=0x55995f2936f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.037+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2bd7b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.037+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2bd7b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.037+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.037+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.037+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.037+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.037+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.037+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.037+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 6e4b281bf038adca43876d3e12fe9ec0ca9bb504d41fc83cde8967061c8852b8 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.037+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114605Z 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:6e4b281bf038adca43876d3e12fe9ec0ca9bb504d41fc83cde8967061c8852b8 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.037+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.037+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.037+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:05.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.037+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.037+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 1fef42c43faaf78dbe4b41a7ba03265d66c99e0fa32d00480d25f4a2ff62740b 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.037+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=1fef42c43faaf78dbe4b41a7ba03265d66c99e0fa32d00480d25f4a2ff62740b 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.037+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.037+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114605Z 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.037+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.037+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=29, curl_handle=0x55995f28b500 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.037+0000 7fb9cb610900 20 run: stack=0x55995f291350 is io blocked 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.037+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2763b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.037+0000 7fb926ffd640 20 link_request req_data=0x55995f27f550 req_data->id=29, curl_handle=0x55995f28b500 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.038+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2763b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.038+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.038+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2763b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.038+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.038+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2763b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.038+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.038+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2dbf80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.038+0000 7fb9cb610900 15 stack 0x55995f2947e0 end 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.038+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is done 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.038+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.038+0000 7fb9cb610900 15 stack 0x55995f28dc40 end 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.038+0000 7fb9cb610900 20 run: stack=0x55995f28dc40 is done 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.038+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.038+0000 7fb9cb610900 20 collect(): s=0x55995f290630 stack=0x55995f2947e0 is complete 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.038+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.038+0000 7fb9cb610900 10 trimming bilogs for bucket=:dipepu-40[4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1]) markers=[,,,,,00000000001.144.6,,,,,], shards=11 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.038+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bc2e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.038+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f287070:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.038+0000 7fb9cb610900 10 trimming bilog shard 5 of :dipepu-40[4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1]) at marker 00000000001.144.6 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.038+0000 7fb9cb610900 20 run: stack=0x55995f290630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.038+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edfc9c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.038+0000 7fb9cb610900 20 trim: trimmed bucket instance dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.038+0000 7fb9cb610900 15 stack 0x55995f18b1a0 end 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.038+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is done 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.038+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.038+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.039+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.039+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.039+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1.5 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.039+0000 7fb9cb610900 20 run: stack=0x55995f28dc40 is io blocked 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.039+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2c84a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.039+0000 7fb9cb610900 4 starting trim on bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.039+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf59b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.039+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf59b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.039+0000 7fb9cb610900 20 enqueued request req=0x55995f282190 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.039+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.039+0000 7fb9cb610900 20 req: 0x55995f282190 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.039+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is io blocked 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.039+0000 7fb98d7fa640 20 dequeued request req=0x55995f282190 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.039+0000 7fb98d7fa640 20 RGWWQ: empty 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.039+0000 7fb98d7fa640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 : miss 2026-03-20T11:46:05.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.039+0000 7fb98d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.039+0000 7fb98d7fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:05.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.039+0000 7fb98d7fa640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 info.flags=0x17 2026-03-20T11:46:05.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.039+0000 7fb98d7fa640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 to cache LRU end 2026-03-20T11:46:05.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.039+0000 7fb98d7fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:05.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.039+0000 7fb98d7fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:05.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.039+0000 7fb98d7fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:46:05.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.039+0000 7fb98d7fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-44 : miss 2026-03-20T11:46:05.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.039+0000 7fb98d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.038 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:05.040+0000 7f127d1ea640 -1 req 11460225817690380754 0.003000004s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.040+0000 7fb98d7fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.040+0000 7fb98d7fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-44 info.flags=0x0 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.040+0000 7fb98d7fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-44 to cache LRU end 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.040+0000 7fb98d7fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-44 : miss 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.040+0000 7fb98d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb98d7fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb98d7fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-44 info.flags=0x0 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb98d7fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-44 to cache LRU end 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb98d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4): adding source pipe: {s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb98d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4): adding dest pipe: {s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb98d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb98d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb98d7fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf59b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf59b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf59b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf59b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2c84a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2bc2e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2c84a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995edfd9d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995edfd9d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 9f726ef0435a40df6a9ee23142982827f25227f94e2e99668e33081b237d97fe 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114605Z 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:9f726ef0435a40df6a9ee23142982827f25227f94e2e99668e33081b237d97fe 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = c43c8ece209028e5ee047cc643a0ecb54a8fabfb7529f4be0380f5bbe96dad94 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c43c8ece209028e5ee047cc643a0ecb54a8fabfb7529f4be0380f5bbe96dad94 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114605Z 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=30, curl_handle=0x55995f27e940 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is io blocked 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb926ffd640 20 link_request req_data=0x55995f28bc10 req_data->id=30, curl_handle=0x55995f27e940 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 15 stack 0x55995f28dc40 end 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 run: stack=0x55995f28dc40 is done 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f287070:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 trim: trimmed bucket instance dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 15 stack 0x55995f290630 end 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 run: stack=0x55995f290630 is done 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 4 starting trim on bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995edf59b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995edf59b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 enqueued request req=0x55995f293860 2026-03-20T11:46:05.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:05.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 req: 0x55995f293860 2026-03-20T11:46:05.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb9cb610900 20 run: stack=0x55995f290630 is io blocked 2026-03-20T11:46:05.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb98dffb640 20 dequeued request req=0x55995f293860 2026-03-20T11:46:05.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb98dffb640 20 RGWWQ: empty 2026-03-20T11:46:05.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.041+0000 7fb98dffb640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 : miss 2026-03-20T11:46:05.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.042+0000 7fb98dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.042+0000 7fb98dffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:05.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.042+0000 7fb98dffb640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 info.flags=0x17 2026-03-20T11:46:05.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.042+0000 7fb98dffb640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 to cache LRU end 2026-03-20T11:46:05.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.042+0000 7fb98dffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:05.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.042+0000 7fb98dffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:05.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.042+0000 7fb98dffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:46:05.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.042+0000 7fb98dffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-17 : miss 2026-03-20T11:46:05.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.042+0000 7fb98dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.042+0000 7fb98dffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.042+0000 7fb98dffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-17 info.flags=0x0 2026-03-20T11:46:05.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.042+0000 7fb98dffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-17 to cache LRU end 2026-03-20T11:46:05.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.042+0000 7fb98dffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-17 : miss 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.042+0000 7fb98dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb98dffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb98dffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-17 info.flags=0x0 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb98dffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-17 to cache LRU end 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb98dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22): adding source pipe: {s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb98dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22): adding dest pipe: {s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb98dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb98dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb98dffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995edf59b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995edf59b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995edf59b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995edf59b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995edf59b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 20 run: stack=0x55995f290630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f2dbf80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f2dbf80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = e7c1674906c7d904afc7695badf9b545f8867f5f60bd0c7c6f672c3ec87b07c7 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114605Z 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:e7c1674906c7d904afc7695badf9b545f8867f5f60bd0c7c6f672c3ec87b07c7 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 2885a465f22fc9b5371e98cfe46f697b04cc088a34e3987a008daea73454686d 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2885a465f22fc9b5371e98cfe46f697b04cc088a34e3987a008daea73454686d 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114605Z 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=31, curl_handle=0x55995f283a50 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb9cb610900 20 run: stack=0x55995f28dc40 is io blocked 2026-03-20T11:46:05.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.043+0000 7fb926ffd640 20 link_request req_data=0x7fb96c0062e0 req_data->id=31, curl_handle=0x55995f283a50 2026-03-20T11:46:05.042 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:05.044+0000 7f12e32b6640 -1 req 2841244807261980663 0.003000004s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:05.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2e1550:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2e1550:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:05.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2bebc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb9cb610900 15 stack 0x55995f2bfaa0 end 2026-03-20T11:46:05.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb9cb610900 20 run: stack=0x55995f2bfaa0 is done 2026-03-20T11:46:05.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f277680:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb9cb610900 20 collect(): s=0x55995f276250 stack=0x55995f2bfaa0 is complete 2026-03-20T11:46:05.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:05.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb9cb610900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:05.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb9cb610900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:05.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f277680:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:05.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb9cb610900 15 stack 0x55995f276250 end 2026-03-20T11:46:05.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb9cb610900 20 stack->operate() returned ret=-22 2026-03-20T11:46:05.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb9cb610900 20 run: stack=0x55995f276250 is done 2026-03-20T11:46:05.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:05.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb9cb610900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:05.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2775d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb9cb610900 4 starting trim on bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:46:05.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f30c8d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f30c8d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb9cb610900 20 enqueued request req=0x55995f2dcbb0 2026-03-20T11:46:05.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:05.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb9cb610900 20 req: 0x55995f2dcbb0 2026-03-20T11:46:05.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb9cb610900 20 run: stack=0x55995f276250 is io blocked 2026-03-20T11:46:05.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb98cff9640 20 dequeued request req=0x55995f2dcbb0 2026-03-20T11:46:05.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb98cff9640 20 RGWWQ: empty 2026-03-20T11:46:05.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb98cff9640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 : miss 2026-03-20T11:46:05.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb98cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb98cff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:05.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb98cff9640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 info.flags=0x17 2026-03-20T11:46:05.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb98cff9640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 to cache LRU end 2026-03-20T11:46:05.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb98cff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:05.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb98cff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:05.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb98cff9640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:46:05.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb98cff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:46:05.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb98cff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-46 : miss 2026-03-20T11:46:05.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.060+0000 7fb98cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb98cff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb98cff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-46 info.flags=0x0 2026-03-20T11:46:05.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb98cff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-46 to cache LRU end 2026-03-20T11:46:05.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb98cff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-46 : miss 2026-03-20T11:46:05.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb98cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb98cff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb98cff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-46 info.flags=0x0 2026-03-20T11:46:05.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb98cff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-46 to cache LRU end 2026-03-20T11:46:05.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb98cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6): adding source pipe: {s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb98cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6): adding dest pipe: {s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb98cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb98cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb98cff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:46:05.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f30c8d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f30c8d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f30c8d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f30c8d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2775d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2bebc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2775d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 20 run: stack=0x55995f276250 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2e1550:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2e1550:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:05.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:05.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 4c6865c281d49e3b4d59721f1c828c62f35ce839a78de468fddf303dfe027550 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114605Z 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:4c6865c281d49e3b4d59721f1c828c62f35ce839a78de468fddf303dfe027550 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = f1f4bd8ce27b518ff56cc0d467c1dc969080e89f67a8dd975d71ad62b319ae43 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=f1f4bd8ce27b518ff56cc0d467c1dc969080e89f67a8dd975d71ad62b319ae43 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114605Z 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=32, curl_handle=0x55995f287050 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb9cb610900 20 run: stack=0x55995f2bfaa0 is io blocked 2026-03-20T11:46:05.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.061+0000 7fb926ffd640 20 link_request req_data=0x55995f2877a0 req_data->id=32, curl_handle=0x55995f287050 2026-03-20T11:46:05.060 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:05.063+0000 7f128ba07640 -1 req 5549582276074982975 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:05.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2bd7b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2bd7b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:05.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2dfab0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb9cb610900 15 stack 0x55995f291350 end 2026-03-20T11:46:05.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb9cb610900 20 run: stack=0x55995f291350 is done 2026-03-20T11:46:05.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f2bfbe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb9cb610900 20 collect(): s=0x55995f2936f0 stack=0x55995f291350 is complete 2026-03-20T11:46:05.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:05.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb9cb610900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:05.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb9cb610900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:05.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f2bfbe0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:05.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb9cb610900 15 stack 0x55995f2936f0 end 2026-03-20T11:46:05.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb9cb610900 20 stack->operate() returned ret=-22 2026-03-20T11:46:05.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb9cb610900 20 run: stack=0x55995f2936f0 is done 2026-03-20T11:46:05.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:05.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb9cb610900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:05.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f2bfbe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb9cb610900 4 starting trim on bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:46:05.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f30c8d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f30c8d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb9cb610900 20 enqueued request req=0x55995f275400 2026-03-20T11:46:05.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:05.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb9cb610900 20 req: 0x55995f275400 2026-03-20T11:46:05.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb9cb610900 20 run: stack=0x55995f2936f0 is io blocked 2026-03-20T11:46:05.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb98e7fc640 20 dequeued request req=0x55995f275400 2026-03-20T11:46:05.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb98e7fc640 20 RGWWQ: empty 2026-03-20T11:46:05.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb98e7fc640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 : miss 2026-03-20T11:46:05.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb98e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb98e7fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:05.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb98e7fc640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 info.flags=0x17 2026-03-20T11:46:05.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb98e7fc640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 to cache LRU end 2026-03-20T11:46:05.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb98e7fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:05.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb98e7fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:05.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb98e7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:46:05.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb98e7fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-37 : miss 2026-03-20T11:46:05.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.081+0000 7fb98e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb98e7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb98e7fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-37 info.flags=0x0 2026-03-20T11:46:05.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb98e7fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-37 to cache LRU end 2026-03-20T11:46:05.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb98e7fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-37 : miss 2026-03-20T11:46:05.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb98e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb98e7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb98e7fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-37 info.flags=0x0 2026-03-20T11:46:05.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb98e7fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-37 to cache LRU end 2026-03-20T11:46:05.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb98e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42): adding source pipe: {s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb98e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42): adding dest pipe: {s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb98e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb98e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb98e7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:46:05.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f30c8d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f30c8d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f30c8d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f30c8d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f2bfbe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2dfab0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f2bfbe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb9cb610900 20 run: stack=0x55995f2936f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f28dd80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f28dd80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.082+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 2f41b84d5cf51f18ff12f30ce808be54ca2e9d90fb0ca8a2448ab4b47960babb 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.083+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114605Z 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:2f41b84d5cf51f18ff12f30ce808be54ca2e9d90fb0ca8a2448ab4b47960babb 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.083+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.083+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.083+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.083+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.083+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = e22cf4c58f12edf91b4413a3ad5a83d662e2f6eee13621a3177bd87cb720db99 2026-03-20T11:46:05.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.083+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=e22cf4c58f12edf91b4413a3ad5a83d662e2f6eee13621a3177bd87cb720db99 2026-03-20T11:46:05.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.083+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.083+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114605Z 2026-03-20T11:46:05.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.083+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:05.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.083+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=33, curl_handle=0x55995f28b500 2026-03-20T11:46:05.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.083+0000 7fb9cb610900 20 run: stack=0x55995f291350 is io blocked 2026-03-20T11:46:05.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.083+0000 7fb926ffd640 20 link_request req_data=0x55995f27f550 req_data->id=33, curl_handle=0x55995f28b500 2026-03-20T11:46:05.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995edfd9d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995edfd9d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:05.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2bc2e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb9cb610900 15 stack 0x55995f2947e0 end 2026-03-20T11:46:05.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is done 2026-03-20T11:46:05.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2c84a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb9cb610900 20 collect(): s=0x55995f18b1a0 stack=0x55995f2947e0 is complete 2026-03-20T11:46:05.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:05.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb9cb610900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:05.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb9cb610900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:05.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2c84a0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:05.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb9cb610900 15 stack 0x55995f18b1a0 end 2026-03-20T11:46:05.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb9cb610900 20 stack->operate() returned ret=-22 2026-03-20T11:46:05.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is done 2026-03-20T11:46:05.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:05.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb9cb610900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:05.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf7790:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb9cb610900 4 starting trim on bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:46:05.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb9cb610900 20 enqueued request req=0x55995f282190 2026-03-20T11:46:05.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:05.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb9cb610900 20 req: 0x55995f282190 2026-03-20T11:46:05.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is io blocked 2026-03-20T11:46:05.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb983fff640 20 dequeued request req=0x55995f282190 2026-03-20T11:46:05.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb983fff640 20 RGWWQ: empty 2026-03-20T11:46:05.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb983fff640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 : miss 2026-03-20T11:46:05.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb983fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb983fff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:46:05.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb983fff640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 info.flags=0x17 2026-03-20T11:46:05.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb983fff640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 to cache LRU end 2026-03-20T11:46:05.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb983fff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:05.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb983fff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:05.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb983fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:46:05.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb983fff640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-1 : miss 2026-03-20T11:46:05.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.085+0000 7fb983fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f2dbf80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb983fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb983fff640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-1 info.flags=0x0 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb983fff640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-1 to cache LRU end 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb983fff640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-1 : miss 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb983fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb983fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb983fff640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-1 info.flags=0x0 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb983fff640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-1 to cache LRU end 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb983fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1): adding source pipe: {s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb983fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1): adding dest pipe: {s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb983fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb983fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb983fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f2dbf80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f2dbf80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f2dbf80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995edf59b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 15 stack 0x55995f28dc40 end 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 20 run: stack=0x55995f28dc40 is done 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 20 collect(): s=0x55995f290630 stack=0x55995f28dc40 is complete 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 10 trimming bilogs for bucket=:dipepu-17[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22]) markers=[,,,,,,,,00000000005.51.18,,], shards=11 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf7790:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2c73e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 10 trimming bilog shard 8 of :dipepu-17[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22]) at marker 00000000005.51.18 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 20 run: stack=0x55995f290630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f2bc2e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf7790:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995edfc9c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22.8 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is io blocked 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995edfc9c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.086+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.087+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.087+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.087+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:05.084 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.087+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 3880912a99fd61b28ee1f454e63162c0a7c02e4025ca5f4491c5cb35ebb030ed 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.087+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114605Z 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:3880912a99fd61b28ee1f454e63162c0a7c02e4025ca5f4491c5cb35ebb030ed 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.087+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.087+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.087+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.087+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.087+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 5140c60ebb6fb5a8365e27e774f4808440a7ede718e9c574f7248cad662e88de 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.087+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5140c60ebb6fb5a8365e27e774f4808440a7ede718e9c574f7248cad662e88de 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.087+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.087+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114605Z 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.087+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.087+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=34, curl_handle=0x55995f283a50 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.087+0000 7fb9cb610900 20 run: stack=0x55995f28dc40 is io blocked 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.087+0000 7fb926ffd640 20 link_request req_data=0x7fb96c0062e0 req_data->id=34, curl_handle=0x55995f283a50 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.088+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.088+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.088+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.088+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f30c8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.088+0000 7fb9cb610900 15 stack 0x55995f2947e0 end 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.088+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is done 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.088+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2c73e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.088+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.088+0000 7fb9cb610900 20 trim: trimmed bucket instance dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.088+0000 7fb9cb610900 15 stack 0x55995f290630 end 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.088+0000 7fb9cb610900 20 run: stack=0x55995f290630 is done 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.088+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.088+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.088+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.088+0000 7fb9cb610900 4 starting trim on bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.088+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.088+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.088+0000 7fb9cb610900 20 enqueued request req=0x55995f293860 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.088+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.088+0000 7fb9cb610900 20 req: 0x55995f293860 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.088+0000 7fb9cb610900 20 run: stack=0x55995f290630 is io blocked 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.088+0000 7fb982ffd640 20 dequeued request req=0x55995f293860 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.088+0000 7fb982ffd640 20 RGWWQ: empty 2026-03-20T11:46:05.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.088+0000 7fb982ffd640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 : miss 2026-03-20T11:46:05.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.088+0000 7fb982ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.089+0000 7fb982ffd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:46:05.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.089+0000 7fb982ffd640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 info.flags=0x17 2026-03-20T11:46:05.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.089+0000 7fb982ffd640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 to cache LRU end 2026-03-20T11:46:05.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.089+0000 7fb982ffd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:05.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.089+0000 7fb982ffd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:05.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.089+0000 7fb982ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:46:05.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.089+0000 7fb982ffd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-7 : miss 2026-03-20T11:46:05.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.089+0000 7fb982ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.086 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:05.089+0000 7f120a905640 -1 req 4683728553286237227 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:05.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.089+0000 7fb982ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.089+0000 7fb982ffd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-7 info.flags=0x0 2026-03-20T11:46:05.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.089+0000 7fb982ffd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-7 to cache LRU end 2026-03-20T11:46:05.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.089+0000 7fb982ffd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-7 : miss 2026-03-20T11:46:05.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.089+0000 7fb982ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.089+0000 7fb982ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.089+0000 7fb982ffd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-7 info.flags=0x0 2026-03-20T11:46:05.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.089+0000 7fb982ffd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-7 to cache LRU end 2026-03-20T11:46:05.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.089+0000 7fb982ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12): adding source pipe: {s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb982ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12): adding dest pipe: {s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb982ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb982ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb982ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f310790:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 20 run: stack=0x55995f290630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995edf59b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995edf59b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = c1d212f3ff3af67fd25d17e792a2ba9e491f454afdbfd2ad9c3ebb3619e080e3 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114605Z 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:c1d212f3ff3af67fd25d17e792a2ba9e491f454afdbfd2ad9c3ebb3619e080e3 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = cd938f462a5a896b3d92c76fc7c50821187f7071c92aa7a0d3bc620358d6b120 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=cd938f462a5a896b3d92c76fc7c50821187f7071c92aa7a0d3bc620358d6b120 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114605Z 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=35, curl_handle=0x55995f27e940 2026-03-20T11:46:05.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is io blocked 2026-03-20T11:46:05.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.090+0000 7fb926ffd640 20 link_request req_data=0x55995f28bc10 req_data->id=35, curl_handle=0x55995f27e940 2026-03-20T11:46:05.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2e1550:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2e1550:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:05.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2bebc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9cb610900 15 stack 0x55995f2bfaa0 end 2026-03-20T11:46:05.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9cb610900 20 run: stack=0x55995f2bfaa0 is done 2026-03-20T11:46:05.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2775d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9cb610900 20 collect(): s=0x55995f276250 stack=0x55995f2bfaa0 is complete 2026-03-20T11:46:05.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:05.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9cb610900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:05.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9cb610900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:05.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2775d0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:05.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9cb610900 15 stack 0x55995f276250 end 2026-03-20T11:46:05.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9cb610900 20 stack->operate() returned ret=-22 2026-03-20T11:46:05.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9cb610900 20 run: stack=0x55995f276250 is done 2026-03-20T11:46:05.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:05.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9cb610900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:05.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995edfa080:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9cb610900 4 starting trim on bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:46:05.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9cb610900 20 enqueued request req=0x55995f2dcbb0 2026-03-20T11:46:05.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:05.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9cb610900 20 req: 0x55995f2dcbb0 2026-03-20T11:46:05.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9827fc640 20 dequeued request req=0x55995f2dcbb0 2026-03-20T11:46:05.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9827fc640 20 RGWWQ: empty 2026-03-20T11:46:05.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9827fc640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 : miss 2026-03-20T11:46:05.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9827fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9cb610900 20 run: stack=0x55995f276250 is io blocked 2026-03-20T11:46:05.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9827fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:05.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9827fc640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 info.flags=0x17 2026-03-20T11:46:05.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9827fc640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 to cache LRU end 2026-03-20T11:46:05.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9827fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:05.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9827fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:05.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9827fc640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:46:05.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9827fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:46:05.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9827fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-48 : miss 2026-03-20T11:46:05.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.104+0000 7fb9827fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.105+0000 7fb9827fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.105+0000 7fb9827fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-48 info.flags=0x0 2026-03-20T11:46:05.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.105+0000 7fb9827fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-48 to cache LRU end 2026-03-20T11:46:05.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.105+0000 7fb9827fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-48 : miss 2026-03-20T11:46:05.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.105+0000 7fb9827fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.105+0000 7fb9827fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.105+0000 7fb9827fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-48 info.flags=0x0 2026-03-20T11:46:05.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.105+0000 7fb9827fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-48 to cache LRU end 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.105+0000 7fb9827fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8): adding source pipe: {s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.105+0000 7fb9827fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8): adding dest pipe: {s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.105+0000 7fb9827fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.105+0000 7fb9827fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.105+0000 7fb9827fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.105+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.105+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.105+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.105+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.105+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995edfa080:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.105+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2bebc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.105+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995edfa080:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.105+0000 7fb9cb610900 20 run: stack=0x55995f276250 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.105+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2e1550:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.105+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2e1550:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.105+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.105+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.105+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.106+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.106+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.106+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.106+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 120c58292137e3288f1775713ed737f623cef5de7aa370c34c70afa9f4f72ea9 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.106+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114605Z 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:120c58292137e3288f1775713ed737f623cef5de7aa370c34c70afa9f4f72ea9 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.106+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.106+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.106+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.106+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.106+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 7db6ff9ffc320700dd27edc7fd08a78ab97541427d60e48c29a3ee164ce06aa8 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.106+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=7db6ff9ffc320700dd27edc7fd08a78ab97541427d60e48c29a3ee164ce06aa8 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.106+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.106+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114605Z 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.106+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.106+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=36, curl_handle=0x55995f287050 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.106+0000 7fb9cb610900 20 run: stack=0x55995f2bfaa0 is io blocked 2026-03-20T11:46:05.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.106+0000 7fb926ffd640 20 link_request req_data=0x55995f2877a0 req_data->id=36, curl_handle=0x55995f287050 2026-03-20T11:46:05.104 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:05.107+0000 7f125799f640 -1 req 10369458668683960918 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:05.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f28dd80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f28dd80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995edfc9c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995edfc9c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:05.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f28dd80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f2bc2e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 15 stack 0x55995f28dc40 end 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 run: stack=0x55995f28dc40 is done 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f28dd80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf7790:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 collect(): s=0x55995f18b1a0 stack=0x55995f28dc40 is complete 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf7790:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 15 stack 0x55995f18b1a0 end 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 stack->operate() returned ret=-22 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is done 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2dfab0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 15 stack 0x55995f291350 end 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 run: stack=0x55995f291350 is done 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f2bfbe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 collect(): s=0x55995f2936f0 stack=0x55995f291350 is complete 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 10 trimming bilogs for bucket=:dipepu-37[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42]) markers=[,,,,,,,,,,00000000001.45.6], shards=11 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf7790:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 4 starting trim on bucket=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f292070:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 10 trimming bilog shard 10 of :dipepu-37[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42]) at marker 00000000001.45.6 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 run: stack=0x55995f2936f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf6d30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf6d30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 enqueued request req=0x55995f2722e0 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 req: 0x55995f2722e0 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is io blocked 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42.10 2026-03-20T11:46:05.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9837fe640 20 dequeued request req=0x55995f2722e0 2026-03-20T11:46:05.129 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9837fe640 20 RGWWQ: empty 2026-03-20T11:46:05.129 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9837fe640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 : miss 2026-03-20T11:46:05.129 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9837fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.130+0000 7fb9cb610900 20 run: stack=0x55995f291350 is io blocked 2026-03-20T11:46:05.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9837fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=386 2026-03-20T11:46:05.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9837fe640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 info.flags=0x17 2026-03-20T11:46:05.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9837fe640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 to cache LRU end 2026-03-20T11:46:05.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9837fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:05.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9837fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:05.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9837fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 2026-03-20T11:46:05.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9837fe640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-6 : miss 2026-03-20T11:46:05.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9837fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9837fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9837fe640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-6 info.flags=0x0 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9837fe640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-6 to cache LRU end 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9837fe640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-6 : miss 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9837fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9837fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9837fe640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-6 info.flags=0x0 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9837fe640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-6 to cache LRU end 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9837fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11): adding source pipe: {s={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9837fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11): adding dest pipe: {s={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9837fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9837fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9837fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf6d30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf6d30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf6d30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf6d30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf7790:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f30d8c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9cb610900 15 stack 0x55995f291350 end 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9cb610900 20 run: stack=0x55995f291350 is done 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f2bc2e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf7790:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f292070:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f28dd80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f2bfbe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9cb610900 20 trim: trimmed bucket instance dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9cb610900 15 stack 0x55995f2936f0 end 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9cb610900 20 run: stack=0x55995f2936f0 is done 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f28dd80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.131+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.131 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = d6756d5bd1821a1e442c5e611738749393f801bcfe3f1de870c49e1375e914cb 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114605Z 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:d6756d5bd1821a1e442c5e611738749393f801bcfe3f1de870c49e1375e914cb 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 4073321b409150cb9fc59bc8583f4fd9e394bd6dcfccf07bcb02a10d5e52a9f3 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=4073321b409150cb9fc59bc8583f4fd9e394bd6dcfccf07bcb02a10d5e52a9f3 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114605Z 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=37, curl_handle=0x55995f283a50 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb9cb610900 20 run: stack=0x55995f28dc40 is io blocked 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f2bfbe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb9cb610900 4 starting trim on bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995edf6d30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995edf6d30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb9cb610900 20 enqueued request req=0x55995f2722e0 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb9cb610900 20 req: 0x55995f2722e0 2026-03-20T11:46:05.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb981ffb640 20 dequeued request req=0x55995f2722e0 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb981ffb640 20 RGWWQ: empty 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb926ffd640 20 link_request req_data=0x7fb96c0062e0 req_data->id=37, curl_handle=0x55995f283a50 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb981ffb640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 : miss 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb981ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb9cb610900 20 run: stack=0x55995f2936f0 is io blocked 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb981ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb981ffb640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 info.flags=0x17 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb981ffb640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 to cache LRU end 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb981ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb981ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb981ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb981ffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-21 : miss 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb981ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb981ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb981ffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-21 info.flags=0x0 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb981ffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-21 to cache LRU end 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb981ffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-21 : miss 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb981ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.132+0000 7fb981ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb981ffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-21 info.flags=0x0 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb981ffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-21 to cache LRU end 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb981ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26): adding source pipe: {s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb981ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26): adding dest pipe: {s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb981ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb981ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb981ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995edf59b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995edf59b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995edf6d30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995edf59b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995edf6d30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995edf59b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995edf6d30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f310790:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 15 stack 0x55995f2947e0 end 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is done 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995edf6d30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 collect(): s=0x55995f290630 stack=0x55995f2947e0 is complete 2026-03-20T11:46:05.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 10 trimming bilogs for bucket=:dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) markers=[,00000000002.6.6,,00000000001.11.6,,00000000001.7.6,00000000002.9.6,00000000001.8.6,00000000001.12.6,00000000001.7.6,], shards=11 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f2bfbe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f310790:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 10 trimming bilog shard 1 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000002.6.6 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 10 trimming bilog shard 3 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.11.6 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 10 trimming bilog shard 5 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.7.6 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 10 trimming bilog shard 6 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000002.9.6 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 10 trimming bilog shard 7 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.8.6 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 10 trimming bilog shard 8 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.12.6 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 10 trimming bilog shard 9 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.7.6 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 run: stack=0x55995f290630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2bf450:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f2bfbe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 run: stack=0x55995f2936f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995edfda00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f2c5d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f28d3c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f2c6680:op=0x55995f2c53a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f2c70b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995edf9390:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f2c5ba0:op=0x55995f2c7ab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995edfc9c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995edfda00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.1 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 run: stack=0x55995f291350 is io blocked 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f2c5d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.3 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 run: stack=0x55995f2be050 is io blocked 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f28d3c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.5 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 run: stack=0x55995f272de0 is io blocked 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f2c6680:op=0x55995f2c53a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.6 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 run: stack=0x55995f2c6680 is io blocked 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f2c70b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.7 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 run: stack=0x55995f28ec10 is io blocked 2026-03-20T11:46:05.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995edf9390:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.8 2026-03-20T11:46:05.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 run: stack=0x55995f30f990 is io blocked 2026-03-20T11:46:05.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f2c5ba0:op=0x55995f2c7ab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.9 2026-03-20T11:46:05.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 run: stack=0x55995f2c5ba0 is io blocked 2026-03-20T11:46:05.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995edfc9c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.133+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:05.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.134+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.134+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:05.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.134+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.136 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.136 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.136 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.136 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.134+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.134+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:05.136 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:05.136 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:05.136 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.136 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.136 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.136 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.136 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.136 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:05.136 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.134+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 876829426a88768683979f5aca0843304c1e6376fbd52daf1d88f267316b9410 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.134+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114605Z 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:876829426a88768683979f5aca0843304c1e6376fbd52daf1d88f267316b9410 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.134+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.134+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.134+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.134+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.134+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = a6fe0f4bade91db2c69bc2322872a2e9fefafaae5142cd2e17a9df96e3a09a2a 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.134+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a6fe0f4bade91db2c69bc2322872a2e9fefafaae5142cd2e17a9df96e3a09a2a 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.134+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.134+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114605Z 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.134+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.134+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=38, curl_handle=0x55995f27e940 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.134+0000 7fb926ffd640 20 link_request req_data=0x55995f28bc10 req_data->id=38, curl_handle=0x55995f27e940 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is io blocked 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 cr:s=0x55995f2c6680:op=0x55995f2c53a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 cr:s=0x55995f2c6680:op=0x55995f2c53a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995edfda00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 cr:s=0x55995f2c6680:op=0x55995f2c53a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995edfda00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 cr:s=0x55995f2c6680:op=0x55995f2c53a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 15 stack 0x55995f2c6680 end 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 run: stack=0x55995f2c6680 is done 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995edfda00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f310790:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 run: stack=0x55995f290630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995edfda00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 15 stack 0x55995f291350 end 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 run: stack=0x55995f291350 is done 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f310790:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 run: stack=0x55995f290630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f2c70b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f2c70b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f2c70b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f2c70b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 15 stack 0x55995f28ec10 end 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 run: stack=0x55995f28ec10 is done 2026-03-20T11:46:05.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f310790:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 run: stack=0x55995f290630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f2c5d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f2c5d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f2c5d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995f2c5d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 15 stack 0x55995f2be050 end 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 run: stack=0x55995f2be050 is done 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f310790:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.135+0000 7fb9cb610900 20 run: stack=0x55995f290630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 cr:s=0x55995f2c5ba0:op=0x55995f2c7ab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 cr:s=0x55995f2c5ba0:op=0x55995f2c7ab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995edf9390:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f28d3c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 cr:s=0x55995f2c5ba0:op=0x55995f2c7ab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995edf9390:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f28d3c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 cr:s=0x55995f2c5ba0:op=0x55995f2c7ab0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 15 stack 0x55995f2c5ba0 end 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 run: stack=0x55995f2c5ba0 is done 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995edf9390:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f28d3c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f310790:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 run: stack=0x55995f290630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995edf9390:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 15 stack 0x55995f30f990 end 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 run: stack=0x55995f30f990 is done 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f28d3c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 15 stack 0x55995f272de0 end 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 run: stack=0x55995f272de0 is done 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f310790:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f310790:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 trim: trimmed bucket instance dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 15 stack 0x55995f290630 end 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 run: stack=0x55995f290630 is done 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 4 starting trim on bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 enqueued request req=0x55995f293860 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 req: 0x55995f293860 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9cb610900 20 run: stack=0x55995f290630 is io blocked 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9817fa640 20 dequeued request req=0x55995f293860 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9817fa640 20 RGWWQ: empty 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9817fa640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 : miss 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.136+0000 7fb9817fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.137+0000 7fb9817fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.137+0000 7fb9817fa640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 info.flags=0x17 2026-03-20T11:46:05.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.137+0000 7fb9817fa640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 to cache LRU end 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.137+0000 7fb9817fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.137+0000 7fb9817fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.137+0000 7fb9817fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.137+0000 7fb9817fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-41 : miss 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.137+0000 7fb9817fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.137+0000 7fb9817fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.137+0000 7fb9817fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-41 info.flags=0x0 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.137+0000 7fb9817fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-41 to cache LRU end 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.137+0000 7fb9817fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-41 : miss 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.137+0000 7fb9817fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9817fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9817fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-41 info.flags=0x0 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9817fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-41 to cache LRU end 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9817fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1): adding source pipe: {s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9817fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1): adding dest pipe: {s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9817fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9817fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9817fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f310790:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 20 run: stack=0x55995f290630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f2767e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f2767e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 259460cad0a6c9dae78c0fdf16ad6ef7ecc8feadbc112242e7e7f2d2ab6be2e8 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114605Z 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:259460cad0a6c9dae78c0fdf16ad6ef7ecc8feadbc112242e7e7f2d2ab6be2e8 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 6c2d4d50364f6ecda5e5616089a6ee37ef3c11d129372694d7cdaf6ac5b7458b 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=6c2d4d50364f6ecda5e5616089a6ee37ef3c11d129372694d7cdaf6ac5b7458b 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114605Z 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:05.139 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=39, curl_handle=0x55995f28b500 2026-03-20T11:46:05.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb9cb610900 20 run: stack=0x55995f30f990 is io blocked 2026-03-20T11:46:05.140 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.138+0000 7fb926ffd640 20 link_request req_data=0x55995f27f550 req_data->id=39, curl_handle=0x55995f28b500 2026-03-20T11:46:05.140 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:05.139+0000 7f12d3a97640 -1 req 5848246056214275519 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2e1550:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2e1550:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2bebc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb9cb610900 15 stack 0x55995f2bfaa0 end 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb9cb610900 20 run: stack=0x55995f2bfaa0 is done 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995edfa080:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb9cb610900 20 collect(): s=0x55995f276250 stack=0x55995f2bfaa0 is complete 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb9cb610900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb9cb610900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995edfa080:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb9cb610900 15 stack 0x55995f276250 end 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb9cb610900 20 stack->operate() returned ret=-22 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb9cb610900 20 run: stack=0x55995f276250 is done 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb9cb610900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995edfa080:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb9cb610900 4 starting trim on bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb9cb610900 20 enqueued request req=0x55995f2dcbb0 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb9cb610900 20 req: 0x55995f2dcbb0 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb9cb610900 20 run: stack=0x55995f276250 is io blocked 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb980ff9640 20 dequeued request req=0x55995f2dcbb0 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb980ff9640 20 RGWWQ: empty 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb980ff9640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 : miss 2026-03-20T11:46:05.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb980ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb980ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:05.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb980ff9640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 info.flags=0x17 2026-03-20T11:46:05.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb980ff9640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 to cache LRU end 2026-03-20T11:46:05.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb980ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:05.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb980ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:05.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb980ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:46:05.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb980ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:46:05.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb980ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-45 : miss 2026-03-20T11:46:05.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb980ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb980ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb980ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-45 info.flags=0x0 2026-03-20T11:46:05.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb980ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-45 to cache LRU end 2026-03-20T11:46:05.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.149+0000 7fb980ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-45 : miss 2026-03-20T11:46:05.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb980ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb980ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb980ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-45 info.flags=0x0 2026-03-20T11:46:05.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb980ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-45 to cache LRU end 2026-03-20T11:46:05.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb980ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5): adding source pipe: {s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb980ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5): adding dest pipe: {s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb980ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb980ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb980ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f292ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995edfa080:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2e1550:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995edfa080:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 20 run: stack=0x55995f276250 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2bebc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2bebc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 77c2a49d7168a743199721c7d726df34bfd61486b29ee0d585c1e6a904359f7f 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114605Z 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:77c2a49d7168a743199721c7d726df34bfd61486b29ee0d585c1e6a904359f7f 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:05.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:05.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:05.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = a1da9f8a216bca36df39592671700118fd837502ee3d0fbbef7e3f7b5b8909b6 2026-03-20T11:46:05.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a1da9f8a216bca36df39592671700118fd837502ee3d0fbbef7e3f7b5b8909b6 2026-03-20T11:46:05.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114605Z 2026-03-20T11:46:05.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:05.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=40, curl_handle=0x55995f287050 2026-03-20T11:46:05.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb9cb610900 20 run: stack=0x55995f2bfaa0 is io blocked 2026-03-20T11:46:05.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.150+0000 7fb926ffd640 20 link_request req_data=0x55995f2877a0 req_data->id=40, curl_handle=0x55995f287050 2026-03-20T11:46:05.149 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:05.151+0000 7f12b1252640 -1 req 15454093599615610618 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:05.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f28dd80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f28dd80:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:05.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f2bc2e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 15 stack 0x55995f28dc40 end 2026-03-20T11:46:05.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 run: stack=0x55995f28dc40 is done 2026-03-20T11:46:05.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf7790:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 collect(): s=0x55995f18b1a0 stack=0x55995f28dc40 is complete 2026-03-20T11:46:05.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:05.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bc2e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11.0 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11.1 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11.2 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11.3 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11.4 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11.5 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11.6 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11.7 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11.8 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11.9 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11.10 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f28dd80:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f287070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995edf59b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f30fc00:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f292ef0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 cr:s=0x55995f2c6680:op=0x55995f2d8d80:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 cr:s=0x55995f2934e0:op=0x55995f291490:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 cr:s=0x55995f273020:op=0x55995f30d8c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 cr:s=0x55995f2c85f0:op=0x55995edf6ce0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 cr:s=0x55995f2d9370:op=0x55995f2c5d20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 cr:s=0x55995f2732e0:op=0x55995f2d84a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f28dd80:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 run: stack=0x55995f28dc40 is io blocked 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f287070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 run: stack=0x55995f272de0 is io blocked 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995edf59b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 run: stack=0x55995f2be050 is io blocked 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f30fc00:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 run: stack=0x55995f28ec10 is io blocked 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f292ef0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 run: stack=0x55995f291350 is io blocked 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 cr:s=0x55995f2c6680:op=0x55995f2d8d80:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 run: stack=0x55995f2c6680 is io blocked 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 cr:s=0x55995f2934e0:op=0x55995f291490:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 run: stack=0x55995f2934e0 is io blocked 2026-03-20T11:46:05.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 cr:s=0x55995f273020:op=0x55995f30d8c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 run: stack=0x55995f273020 is io blocked 2026-03-20T11:46:05.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 cr:s=0x55995f2c85f0:op=0x55995edf6ce0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 run: stack=0x55995f2c85f0 is io blocked 2026-03-20T11:46:05.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 cr:s=0x55995f2d9370:op=0x55995f2c5d20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 run: stack=0x55995f2d9370 is io blocked 2026-03-20T11:46:05.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 cr:s=0x55995f2732e0:op=0x55995f2d84a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.173+0000 7fb9cb610900 20 run: stack=0x55995f2732e0 is io blocked 2026-03-20T11:46:05.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f2c85f0:op=0x55995edf6ce0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f2c85f0:op=0x55995edf6ce0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f2c85f0:op=0x55995edf6ce0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f2c85f0:op=0x55995edf6ce0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 15 stack 0x55995f2c85f0 end 2026-03-20T11:46:05.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 run: stack=0x55995f2c85f0 is done 2026-03-20T11:46:05.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bc2e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:05.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f2c6680:op=0x55995f2d8d80:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f2c6680:op=0x55995f2d8d80:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f2c6680:op=0x55995f2d8d80:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f2c6680:op=0x55995f2d8d80:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 15 stack 0x55995f2c6680 end 2026-03-20T11:46:05.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 run: stack=0x55995f2c6680 is done 2026-03-20T11:46:05.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bc2e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:05.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f292ef0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f292ef0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f292ef0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f292ef0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 15 stack 0x55995f291350 end 2026-03-20T11:46:05.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 run: stack=0x55995f291350 is done 2026-03-20T11:46:05.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bc2e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:05.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f273020:op=0x55995f30d8c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f273020:op=0x55995f30d8c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f30fc00:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f2934e0:op=0x55995f291490:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f273020:op=0x55995f30d8c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f30fc00:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f2934e0:op=0x55995f291490:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f273020:op=0x55995f30d8c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 15 stack 0x55995f273020 end 2026-03-20T11:46:05.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 run: stack=0x55995f273020 is done 2026-03-20T11:46:05.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f30fc00:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f2934e0:op=0x55995f291490:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bc2e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f30fc00:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 15 stack 0x55995f28ec10 end 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 run: stack=0x55995f28ec10 is done 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f2934e0:op=0x55995f291490:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 15 stack 0x55995f2934e0 end 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 run: stack=0x55995f2934e0 is done 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bc2e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bc2e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.175+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f287070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f287070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f287070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f287070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 15 stack 0x55995f272de0 end 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 run: stack=0x55995f272de0 is done 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bc2e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995edf59b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995edf59b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 cr:s=0x55995f2d9370:op=0x55995f2c5d20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f28dd80:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995edf59b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 cr:s=0x55995f2d9370:op=0x55995f2c5d20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f28dd80:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 cr:s=0x55995f2be050:op=0x55995edf59b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 15 stack 0x55995f2be050 end 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 run: stack=0x55995f2be050 is done 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 cr:s=0x55995f2d9370:op=0x55995f2c5d20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f28dd80:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bc2e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 cr:s=0x55995f2d9370:op=0x55995f2c5d20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 15 stack 0x55995f2d9370 end 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 run: stack=0x55995f2d9370 is done 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 cr:s=0x55995f28dc40:op=0x55995f28dd80:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 15 stack 0x55995f28dc40 end 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 run: stack=0x55995f28dc40 is done 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bc2e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bc2e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 cr:s=0x55995f2732e0:op=0x55995f2d84a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 cr:s=0x55995f2732e0:op=0x55995f2d84a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 cr:s=0x55995f2732e0:op=0x55995f2d84a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 cr:s=0x55995f2732e0:op=0x55995f2d84a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 15 stack 0x55995f2732e0 end 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 run: stack=0x55995f2732e0 is done 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2bc2e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf7790:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f292ef0:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f292ef0:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:05.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 enqueued request req=0x55995edf6bc0 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 req: 0x55995edf6bc0 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is io blocked 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb97fff7640 20 dequeued request req=0x55995edf6bc0 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.176+0000 7fb97fff7640 20 RGWWQ: empty 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.178+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995edfc9c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.178+0000 7fb97fff7640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 info.flags=0x11 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.178+0000 7fb97fff7640 10 rgw async rados processor: moving a2.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 to cache LRU end 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.178+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995edfc9c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.178+0000 7fb97fff7640 10 rgw async rados processor: distributing notification oid=a2.rgw.control:notify.5 cni=[op: 0, obj: a2.rgw.meta:root:.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11, ofs0, ns] 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.178+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995edfc9c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.178+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995edfc9c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.178+0000 7fb9cb610900 20 cr:s=0x55995f2947e0:op=0x55995f2bf450:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.178+0000 7fb9cb610900 15 stack 0x55995f2947e0 end 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.178+0000 7fb9cb610900 20 run: stack=0x55995f2947e0 is done 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.178+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f2bfbe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.178+0000 7fb9cb610900 20 collect(): s=0x55995f2936f0 stack=0x55995f2947e0 is complete 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.178+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.178+0000 7fb9cb610900 10 trimming bilogs for bucket=:dipepu-21[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26]) markers=[,,,,,,,,00000000005.84.14,,], shards=11 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.178+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f30fc00:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.178+0000 7fb9cb610900 10 trimming bilog shard 8 of :dipepu-21[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26]) at marker 00000000005.84.14 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.178+0000 7fb9cb610900 20 run: stack=0x55995f2936f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.178+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2c5d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.178+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2c5d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.178+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26.8 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.178+0000 7fb9cb610900 20 run: stack=0x55995f272de0 is io blocked 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.178+0000 7fb9a2ffd640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 343597383758 cookie 94117215055120 notifier 6376 bl.length()=565 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.178+0000 7fb9a2ffd640 10 rgw watcher librados: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 info.flags=0x11 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.178+0000 7fb9a2ffd640 10 rgw watcher librados: moving a2.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 to cache LRU end 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.179+0000 7fb97fff7640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-6[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11]): orig_sources= new_sources= 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.179+0000 7fb97fff7640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-6[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11]): potential sources added=[] removed=[] 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.179+0000 7fb97fff7640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-6[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11]): orig_dests= new_dests= 2026-03-20T11:46:05.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.179+0000 7fb97fff7640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-6[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11]): potential dests added=[] removed=[] 2026-03-20T11:46:05.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.179+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f292ef0:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:05.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.179+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f292ef0:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:05.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.179+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f292ef0:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:05.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.179+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f292ef0:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:05.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.179+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf7790:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.179+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edfd5a0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:05.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.179+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edfd5a0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:05.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.179+0000 7fb9cb610900 20 enqueued request req=0x55995ed93f40 2026-03-20T11:46:05.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.179+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:05.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.179+0000 7fb9cb610900 20 req: 0x55995ed93f40 2026-03-20T11:46:05.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.179+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is io blocked 2026-03-20T11:46:05.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.179+0000 7fb9807f8640 20 dequeued request req=0x55995ed93f40 2026-03-20T11:46:05.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.179+0000 7fb9807f8640 20 RGWWQ: empty 2026-03-20T11:46:05.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2c5d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2c5d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2c5d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2c5d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb9cb610900 15 stack 0x55995f272de0 end 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb9cb610900 20 run: stack=0x55995f272de0 is done 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f30fc00:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb9cb610900 20 cr:s=0x55995f2936f0:op=0x55995f2bfbe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb9cb610900 20 trim: trimmed bucket instance dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb9cb610900 15 stack 0x55995f2936f0 end 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb9cb610900 20 run: stack=0x55995f2936f0 is done 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2bfbe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb9cb610900 4 starting trim on bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2c5d20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2c5d20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb9cb610900 20 enqueued request req=0x55995f27b860 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb9cb610900 20 req: 0x55995f27b860 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb9cb610900 20 run: stack=0x55995f272de0 is io blocked 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb97f7f6640 20 dequeued request req=0x55995f27b860 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb97f7f6640 20 RGWWQ: empty 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb97f7f6640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 : miss 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb97f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb9807f8640 10 rgw async rados processor: removing a2.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 from cache 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb9807f8640 10 rgw async rados processor: distributing notification oid=a2.rgw.control:notify.5 cni=[op: 1, obj: a2.rgw.meta:root:.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11, ofs0, ns] 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb97f7f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb97f7f6640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 info.flags=0x17 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb97f7f6640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 to cache LRU end 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb97f7f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb97f7f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb97f7f6640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb97f7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb97f7f6640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-33 : miss 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb97f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb97f7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb97f7f6640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-33 info.flags=0x0 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb97f7f6640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-33 to cache LRU end 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb97f7f6640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-33 : miss 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb97f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.180+0000 7fb9a2ffd640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 343597383759 cookie 94117215055120 notifier 6376 bl.length()=205 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f2767e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f2767e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f310790:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 15 stack 0x55995f30f990 end 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 run: stack=0x55995f30f990 is done 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 collect(): s=0x55995f290630 stack=0x55995f30f990 is complete 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 cr:s=0x55995f290630:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 15 stack 0x55995f290630 end 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 stack->operate() returned ret=-22 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 run: stack=0x55995f290630 is done 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edfd5a0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97f7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97f7f6640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-33 info.flags=0x0 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edfd5a0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 4 starting trim on bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97f7f6640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-33 to cache LRU end 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edfd5a0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f28d3c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edfd5a0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f28d3c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 enqueued request req=0x55995f282190 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 req: 0x55995f282190 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38): adding source pipe: {s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 run: stack=0x55995f30f990 is io blocked 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995edf7790:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 trim: trimmed bucket instance dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 15 stack 0x55995f18b1a0 end 2026-03-20T11:46:05.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38): adding dest pipe: {s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is done 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2789a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 4 starting trim on bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 enqueued request req=0x55995f294580 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 req: 0x55995f282190 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 req: 0x55995f294580 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is io blocked 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97f7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97f7f6640 20 dequeued request req=0x55995f282190 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97f7f6640 20 RGWWQ: 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97f7f6640 20 req: 0x55995f294580 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97f7f6640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 : miss 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97eff5640 20 dequeued request req=0x55995f294580 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97eff5640 20 RGWWQ: empty 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97eff5640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 : miss 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97eff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2c5d20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2c5d20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2c5d20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2c5d20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2bfbe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 cr:s=0x55995f2934e0:op=0x55995f2bc2e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2bfbe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 run: stack=0x55995f272de0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 cr:s=0x55995f2934e0:op=0x55995f2bf450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 cr:s=0x55995f2934e0:op=0x55995f2bf450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = b01bd332b35e1891081d0b6f7e787afe27cdf37a33364b4589245b0a182c09e9 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114605Z 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:b01bd332b35e1891081d0b6f7e787afe27cdf37a33364b4589245b0a182c09e9 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 564769db720a87fda081be5c5d6acfed6cff3f5708e2e9bfaf0a5b0e8dabe1e0 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=564769db720a87fda081be5c5d6acfed6cff3f5708e2e9bfaf0a5b0e8dabe1e0 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114605Z 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=41, curl_handle=0x55995f28b500 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb9cb610900 20 run: stack=0x55995f2934e0 is io blocked 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb926ffd640 20 link_request req_data=0x7fb95c007fd0 req_data->id=41, curl_handle=0x55995f28b500 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97eff5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97eff5640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 info.flags=0x17 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97eff5640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 to cache LRU end 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97eff5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97eff5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97eff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97eff5640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-30 : miss 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97eff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97f7f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97f7f6640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 info.flags=0x17 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97f7f6640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 to cache LRU end 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97f7f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97f7f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97f7f6640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97f7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97f7f6640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-47 : miss 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.181+0000 7fb97f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb97eff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb97eff5640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-30 info.flags=0x0 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb97eff5640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-30 to cache LRU end 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb97eff5640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-30 : miss 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb97eff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb97f7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb97f7f6640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-47 info.flags=0x0 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb97f7f6640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-47 to cache LRU end 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb97f7f6640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-47 : miss 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb97f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb97eff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb97eff5640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-30 info.flags=0x0 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb97eff5640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-30 to cache LRU end 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb97eff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35): adding source pipe: {s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb97eff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35): adding dest pipe: {s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb97eff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb97eff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb97eff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb97f7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb97f7f6640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-47 info.flags=0x0 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb97f7f6640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-47 to cache LRU end 2026-03-20T11:46:05.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb97f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7): adding source pipe: {s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb97f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7): adding dest pipe: {s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2789a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb97f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb97f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995edfdd80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2789a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995edfc9c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995edfc9c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb97f7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = bb7932837a0ccab0f16b1543a40deed3ce43e82eb8687fcbac3c59fa2c365911 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114605Z 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:bb7932837a0ccab0f16b1543a40deed3ce43e82eb8687fcbac3c59fa2c365911 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 3fc7f4f8258db934b09a21d0c95cb93fa6a7d3b167ccab518b89e92478baddcf 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=3fc7f4f8258db934b09a21d0c95cb93fa6a7d3b167ccab518b89e92478baddcf 2026-03-20T11:46:05.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114605Z 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=42, curl_handle=0x55995f27e940 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 run: stack=0x55995f28ec10 is io blocked 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f28d3c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f28d3c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f28d3c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f28d3c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 cr:s=0x55995f273020:op=0x55995f310790:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 run: stack=0x55995f30f990 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 cr:s=0x55995f273020:op=0x55995f30cbb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 cr:s=0x55995f273020:op=0x55995f30cbb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb926ffd640 20 link_request req_data=0x55995f27f550 req_data->id=42, curl_handle=0x55995f27e940 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 410e35afb16a4de6feec21145d800c37163e314f68ee7d537de4ba06e76c6d1b 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114605Z 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:410e35afb16a4de6feec21145d800c37163e314f68ee7d537de4ba06e76c6d1b 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 6093241d552599c0e4483db1fd3136bbab399c202d37633060d2764c3e8a6935 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=6093241d552599c0e4483db1fd3136bbab399c202d37633060d2764c3e8a6935 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114605Z 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=43, curl_handle=0x55995f283a50 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb9cb610900 20 run: stack=0x55995f273020 is io blocked 2026-03-20T11:46:05.182 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.182+0000 7fb926ffd640 20 link_request req_data=0x55995f28bc10 req_data->id=43, curl_handle=0x55995f283a50 2026-03-20T11:46:05.182 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:05.184+0000 7f126b1c6640 -1 req 7720110196175399383 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:05.182 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:05.184+0000 7f12781e0640 -1 req 6120749668738669547 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:05.182 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:05.184+0000 7f122693d640 -1 req 2118103641553693270 0.003000004s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2bebc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2bebc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2e1550:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb9cb610900 15 stack 0x55995f2bfaa0 end 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb9cb610900 20 run: stack=0x55995f2bfaa0 is done 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995edfa080:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb9cb610900 20 collect(): s=0x55995f276250 stack=0x55995f2bfaa0 is complete 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb9cb610900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb9cb610900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995edfa080:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb9cb610900 15 stack 0x55995f276250 end 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb9cb610900 20 stack->operate() returned ret=-22 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb9cb610900 20 run: stack=0x55995f276250 is done 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb9cb610900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995edfa080:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb9cb610900 4 starting trim on bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb9cb610900 20 enqueued request req=0x55995f2dcbb0 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb9cb610900 20 req: 0x55995f2dcbb0 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb97d7f2640 20 dequeued request req=0x55995f2dcbb0 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb97d7f2640 20 RGWWQ: empty 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb97d7f2640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 : miss 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb97d7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb9cb610900 20 run: stack=0x55995f276250 is io blocked 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb97d7f2640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb97d7f2640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 info.flags=0x17 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb97d7f2640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 to cache LRU end 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb97d7f2640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb97d7f2640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb97d7f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb97d7f2640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-20 : miss 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb97d7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb97d7f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb97d7f2640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-20 info.flags=0x0 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb97d7f2640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-20 to cache LRU end 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb97d7f2640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-20 : miss 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.193+0000 7fb97d7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb97d7f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb97d7f2640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-20 info.flags=0x0 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb97d7f2640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-20 to cache LRU end 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb97d7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25): adding source pipe: {s={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb97d7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25): adding dest pipe: {s={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb97d7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb97d7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb97d7f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995edfa080:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2e1550:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995edfa080:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 20 run: stack=0x55995f276250 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2bebc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2bebc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = eaf502c6ec80c533648b0a4b8860ec03d7e8fc79c0c525a6ed61a41f5108ba1c 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114605Z 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:eaf502c6ec80c533648b0a4b8860ec03d7e8fc79c0c525a6ed61a41f5108ba1c 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = ff00e02b750d3530206b73dfe33ba8602fc6a265cb914f6833b724d9e5e34124 2026-03-20T11:46:05.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=ff00e02b750d3530206b73dfe33ba8602fc6a265cb914f6833b724d9e5e34124 2026-03-20T11:46:05.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114605Z 2026-03-20T11:46:05.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:05.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=44, curl_handle=0x55995f287050 2026-03-20T11:46:05.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb9cb610900 20 run: stack=0x55995f2bfaa0 is io blocked 2026-03-20T11:46:05.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.194+0000 7fb926ffd640 20 link_request req_data=0x55995f2877a0 req_data->id=44, curl_handle=0x55995f287050 2026-03-20T11:46:05.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 cr:s=0x55995f2934e0:op=0x55995f2bf450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 cr:s=0x55995f2934e0:op=0x55995f2bf450:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:05.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 cr:s=0x55995f2934e0:op=0x55995f2bc2e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 15 stack 0x55995f2934e0 end 2026-03-20T11:46:05.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 run: stack=0x55995f2934e0 is done 2026-03-20T11:46:05.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995edfc9c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995edfc9c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:05.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 cr:s=0x55995f273020:op=0x55995f30cbb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 cr:s=0x55995f273020:op=0x55995f30cbb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:05.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2bfbe0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 collect(): s=0x55995f272de0 stack=0x55995f2934e0 is complete 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2bfbe0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 15 stack 0x55995f272de0 end 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 stack->operate() returned ret=-22 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 run: stack=0x55995f272de0 is done 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995edfdd80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 15 stack 0x55995f28ec10 end 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 run: stack=0x55995f28ec10 is done 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 cr:s=0x55995f273020:op=0x55995f310790:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 15 stack 0x55995f273020 end 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 run: stack=0x55995f273020 is done 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2789a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 collect(): s=0x55995f18b1a0 stack=0x55995f28ec10 is complete 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2789a0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 15 stack 0x55995f18b1a0 end 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 stack->operate() returned ret=-22 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is done 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 collect(): s=0x55995f30f990 stack=0x55995f273020 is complete 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 15 stack 0x55995f30f990 end 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 stack->operate() returned ret=-22 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 run: stack=0x55995f30f990 is done 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2c53a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 4 starting trim on bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:05.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 4 starting trim on bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 enqueued request req=0x55995f2d5f10 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 req: 0x55995f2d5f10 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 run: stack=0x55995f272de0 is io blocked 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f28d3c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f278960:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 4 starting trim on bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f28d3c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 enqueued request req=0x55995f30c9c0 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 req: 0x55995f2d5f10 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 req: 0x55995f30c9c0 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 run: stack=0x55995f30f990 is io blocked 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2c87a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2c87a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97dff3640 20 dequeued request req=0x55995f2d5f10 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97dff3640 20 RGWWQ: 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97dff3640 20 req: 0x55995f30c9c0 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97cff1640 20 dequeued request req=0x55995f30c9c0 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97cff1640 20 RGWWQ: empty 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97cff1640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 : miss 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97dff3640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 : miss 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97cff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97dff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 enqueued request req=0x55995f290f50 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 req: 0x55995f290f50 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is io blocked 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97c7f0640 20 dequeued request req=0x55995f290f50 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97c7f0640 20 RGWWQ: empty 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97c7f0640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 : miss 2026-03-20T11:46:05.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97c7f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97cff1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97cff1640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 info.flags=0x17 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97cff1640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 to cache LRU end 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97cff1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97cff1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97cff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97cff1640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-2 : miss 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97cff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97c7f0640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97c7f0640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 info.flags=0x17 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97c7f0640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 to cache LRU end 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97c7f0640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97c7f0640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97c7f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97c7f0640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-29 : miss 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.225+0000 7fb97c7f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97dff3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97dff3640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 info.flags=0x17 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97dff3640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 to cache LRU end 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97dff3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97dff3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97dff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97dff3640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-42 : miss 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97dff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97c7f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97c7f0640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-29 info.flags=0x0 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97c7f0640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-29 to cache LRU end 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97c7f0640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-29 : miss 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97c7f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97cff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97cff1640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-2 info.flags=0x0 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97cff1640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-2 to cache LRU end 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97cff1640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-2 : miss 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97cff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97dff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97dff3640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-42 info.flags=0x0 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97dff3640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-42 to cache LRU end 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97dff3640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-42 : miss 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97dff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97c7f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97c7f0640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-29 info.flags=0x0 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97c7f0640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-29 to cache LRU end 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97c7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34): adding source pipe: {s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97c7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34): adding dest pipe: {s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97c7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97c7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97c7f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:46:05.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97dff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97dff3640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-42 info.flags=0x0 2026-03-20T11:46:05.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97dff3640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-42 to cache LRU end 2026-03-20T11:46:05.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97cff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97cff1640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-2 info.flags=0x0 2026-03-20T11:46:05.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2): adding source pipe: {s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97cff1640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-2 to cache LRU end 2026-03-20T11:46:05.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2): adding dest pipe: {s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97cff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2): adding source pipe: {s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97cff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2): adding dest pipe: {s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97cff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97cff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97dff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:46:05.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb97cff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:46:05.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2c87a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2c87a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f28d3c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2c87a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f28d3c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2c87a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f28d3c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f278960:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2dbf80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f28d3c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 cr:s=0x55995f273020:op=0x55995f2bc2e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f278960:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2c53a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 cr:s=0x55995f273020:op=0x55995edfc9c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f2bf450:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2c53a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 run: stack=0x55995f272de0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 cr:s=0x55995f2934e0:op=0x55995f28b540:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 run: stack=0x55995f30f990 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 cr:s=0x55995f273020:op=0x55995edfc9c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = ad5abc5011f5712ff0c57d5f5eb8a8953ea96ed04b1a37c27aa5c05fb334df2c 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114605Z 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:ad5abc5011f5712ff0c57d5f5eb8a8953ea96ed04b1a37c27aa5c05fb334df2c 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 2b569c30d03b61e68eb6656347c1116dd8ae98c3919d1b964490dc65b9ea4987 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2b569c30d03b61e68eb6656347c1116dd8ae98c3919d1b964490dc65b9ea4987 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114605Z 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=45, curl_handle=0x55995f283a50 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 run: stack=0x55995f273020 is io blocked 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f293f20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 cr:s=0x55995f2934e0:op=0x55995f2e5350:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f293f20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb926ffd640 20 link_request req_data=0x55995f28bc10 req_data->id=45, curl_handle=0x55995f283a50 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.228 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 6f15b5c3c6564ca994627e7b79d21316c2e8fa6113b64b13a8bf33f962973c31 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114605Z 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:6f15b5c3c6564ca994627e7b79d21316c2e8fa6113b64b13a8bf33f962973c31 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = b81be5de44edd5bb299011733b4f5c8d4eadbdf060a6a67f36cae1a956eceffe 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b81be5de44edd5bb299011733b4f5c8d4eadbdf060a6a67f36cae1a956eceffe 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114605Z 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=46, curl_handle=0x55995f27e940 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 run: stack=0x55995f28ec10 is io blocked 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 cr:s=0x55995f2934e0:op=0x55995f2e5350:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb926ffd640 20 link_request req_data=0x55995f27f550 req_data->id=46, curl_handle=0x55995f27e940 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.226+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.227+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.227+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.227+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.227+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 751ffdb481f51a3f1159a38594cd5917f0731590ceefe61abd9d27cc2a6fefe4 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.227+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114605Z 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:751ffdb481f51a3f1159a38594cd5917f0731590ceefe61abd9d27cc2a6fefe4 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.227+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.227+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.227+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.227+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.227+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = a1bad7574ef730e9fed655754bad59349cd6f707a46f8ff1c2522cae82dd49cd 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.227+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a1bad7574ef730e9fed655754bad59349cd6f707a46f8ff1c2522cae82dd49cd 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.227+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.227+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114605Z 2026-03-20T11:46:05.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.227+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:05.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.227+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=47, curl_handle=0x55995f28b500 2026-03-20T11:46:05.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.227+0000 7fb9cb610900 20 run: stack=0x55995f2934e0 is io blocked 2026-03-20T11:46:05.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.227+0000 7fb926ffd640 20 link_request req_data=0x7fb95c007fd0 req_data->id=47, curl_handle=0x55995f28b500 2026-03-20T11:46:05.230 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:05.228+0000 7f129c228640 -1 req 6623143587025877637 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:05.230 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:05.228+0000 7f12cc288640 -1 req 2558586195527339704 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:05.230 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:05.228+0000 7f12d629c640 -1 req 8476444924197667010 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:05.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.237+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2bebc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.237+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2bebc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.237+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2bebc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.237+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2bebc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.237+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2e1550:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.237+0000 7fb9cb610900 15 stack 0x55995f2bfaa0 end 2026-03-20T11:46:05.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.237+0000 7fb9cb610900 20 run: stack=0x55995f2bfaa0 is done 2026-03-20T11:46:05.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.237+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995edfa080:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.237+0000 7fb9cb610900 20 collect(): s=0x55995f276250 stack=0x55995f2bfaa0 is complete 2026-03-20T11:46:05.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.237+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:05.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.237+0000 7fb9cb610900 10 trimming bilogs for bucket=:dipepu-20[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25]) markers=[,,,,,,,,00000000001.41.14,,], shards=11 2026-03-20T11:46:05.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.237+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f292ef0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.237+0000 7fb9cb610900 10 trimming bilog shard 8 of :dipepu-20[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25]) at marker 00000000001.41.14 2026-03-20T11:46:05.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.237+0000 7fb9cb610900 20 run: stack=0x55995f276250 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.237+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f285b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.237+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f285b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.237+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25.8 2026-03-20T11:46:05.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.237+0000 7fb9cb610900 20 run: stack=0x55995f2bfaa0 is io blocked 2026-03-20T11:46:05.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.238+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f285b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.238+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f285b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.238+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f285b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.238+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f285b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.238+0000 7fb9cb610900 15 stack 0x55995f2bfaa0 end 2026-03-20T11:46:05.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.238+0000 7fb9cb610900 20 run: stack=0x55995f2bfaa0 is done 2026-03-20T11:46:05.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.238+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f292ef0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.238+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995edfa080:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.238+0000 7fb9cb610900 20 trim: trimmed bucket instance dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:05.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.238+0000 7fb9cb610900 15 stack 0x55995f276250 end 2026-03-20T11:46:05.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.238+0000 7fb9cb610900 20 run: stack=0x55995f276250 is done 2026-03-20T11:46:05.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.238+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:05.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.238+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.238+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995edfa080:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.238+0000 7fb9cb610900 4 starting trim on bucket=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 2026-03-20T11:46:05.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.238+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2773c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.238+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2773c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.238+0000 7fb9cb610900 20 enqueued request req=0x55995f2dcbb0 2026-03-20T11:46:05.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.238+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:05.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.238+0000 7fb9cb610900 20 req: 0x55995f2dcbb0 2026-03-20T11:46:05.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.238+0000 7fb9cb610900 20 run: stack=0x55995f276250 is io blocked 2026-03-20T11:46:05.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.238+0000 7fb97bfef640 20 dequeued request req=0x55995f2dcbb0 2026-03-20T11:46:05.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.238+0000 7fb97bfef640 20 RGWWQ: empty 2026-03-20T11:46:05.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.238+0000 7fb97bfef640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 : miss 2026-03-20T11:46:05.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.238+0000 7fb97bfef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.238+0000 7fb97bfef640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=408 2026-03-20T11:46:05.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb97bfef640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 info.flags=0x17 2026-03-20T11:46:05.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb97bfef640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 to cache LRU end 2026-03-20T11:46:05.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb97bfef640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:05.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb97bfef640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=171 2026-03-20T11:46:05.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb97bfef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 2026-03-20T11:46:05.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb97bfef640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.testx/tenanted-bucket : miss 2026-03-20T11:46:05.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb97bfef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb97bfef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb97bfef640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.testx/tenanted-bucket info.flags=0x0 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb97bfef640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.testx/tenanted-bucket to cache LRU end 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb97bfef640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.testx/tenanted-bucket : miss 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb97bfef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb97bfef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb97bfef640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.testx/tenanted-bucket info.flags=0x0 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb97bfef640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.testx/tenanted-bucket to cache LRU end 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb97bfef640 20 rgw async rados processor: reflect(): flow manager (bucket=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3): adding source pipe: {s={b=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb97bfef640 20 rgw async rados processor: reflect(): flow manager (bucket=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3): adding dest pipe: {s={b=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb97bfef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb97bfef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb97bfef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2773c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2773c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2773c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2773c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995edfa080:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2e1550:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995edfa080:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 20 run: stack=0x55995f276250 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2bebc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2bebc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.237 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.238 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:05.238 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.238 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 311a2f5ac1010d7ab8458702f2d20a7f1c0a02c57f15ab7fb19331669e711583 2026-03-20T11:46:05.238 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:05.238 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114605Z 2026-03-20T11:46:05.238 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:05.238 INFO:teuthology.orchestra.run.vm02.stderr:311a2f5ac1010d7ab8458702f2d20a7f1c0a02c57f15ab7fb19331669e711583 2026-03-20T11:46:05.238 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:05.238 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:05.238 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:05.238 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:05.238 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 549909dc7b9da9c371b9869969a55dac2e8cee6f8130fc5e11b2972e13e5e41f 2026-03-20T11:46:05.238 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=549909dc7b9da9c371b9869969a55dac2e8cee6f8130fc5e11b2972e13e5e41f 2026-03-20T11:46:05.238 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.238 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114605Z 2026-03-20T11:46:05.238 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:05.238 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=48, curl_handle=0x55995f287050 2026-03-20T11:46:05.238 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb9cb610900 20 run: stack=0x55995f2bfaa0 is io blocked 2026-03-20T11:46:05.238 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.239+0000 7fb926ffd640 20 link_request req_data=0x55995f2877a0 req_data->id=48, curl_handle=0x55995f287050 2026-03-20T11:46:05.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 cr:s=0x55995f273020:op=0x55995edfc9c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 cr:s=0x55995f273020:op=0x55995edfc9c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:05.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 cr:s=0x55995f273020:op=0x55995f2bc2e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 15 stack 0x55995f273020 end 2026-03-20T11:46:05.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 run: stack=0x55995f273020 is done 2026-03-20T11:46:05.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f293f20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f293f20:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:05.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 cr:s=0x55995f2934e0:op=0x55995f2e5350:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 cr:s=0x55995f2934e0:op=0x55995f2e5350:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:05.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f278960:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 collect(): s=0x55995f18b1a0 stack=0x55995f273020 is complete 2026-03-20T11:46:05.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:05.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:05.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:05.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f278960:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 15 stack 0x55995f18b1a0 end 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 stack->operate() returned ret=-22 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is done 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f2bf450:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 15 stack 0x55995f28ec10 end 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 run: stack=0x55995f28ec10 is done 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 cr:s=0x55995f2934e0:op=0x55995f28b540:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 15 stack 0x55995f2934e0 end 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 run: stack=0x55995f2934e0 is done 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2c53a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 collect(): s=0x55995f272de0 stack=0x55995f28ec10 is complete 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2c53a0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 15 stack 0x55995f272de0 end 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 stack->operate() returned ret=-22 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 run: stack=0x55995f272de0 is done 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 collect(): s=0x55995f30f990 stack=0x55995f2934e0 is complete 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 cr:s=0x55995f30f990:op=0x55995f2e23f0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 15 stack 0x55995f30f990 end 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 stack->operate() returned ret=-22 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 run: stack=0x55995f30f990 is done 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f278960:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 4 starting trim on bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2773c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:05.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2773c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 enqueued request req=0x55995f293860 2026-03-20T11:46:05.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:05.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 req: 0x55995f293860 2026-03-20T11:46:05.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is io blocked 2026-03-20T11:46:05.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb97b7ee640 20 dequeued request req=0x55995f293860 2026-03-20T11:46:05.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb97b7ee640 20 RGWWQ: empty 2026-03-20T11:46:05.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb97b7ee640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 : miss 2026-03-20T11:46:05.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb97b7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb97b7ee640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:05.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb97b7ee640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 info.flags=0x17 2026-03-20T11:46:05.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb97b7ee640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 to cache LRU end 2026-03-20T11:46:05.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb97b7ee640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:05.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb97b7ee640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:05.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb97b7ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:05.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb97b7ee640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-23 : miss 2026-03-20T11:46:05.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.269+0000 7fb97b7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb97b7ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb97b7ee640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-23 info.flags=0x0 2026-03-20T11:46:05.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb97b7ee640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-23 to cache LRU end 2026-03-20T11:46:05.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb97b7ee640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-23 : miss 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb97b7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb97b7ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb97b7ee640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-23 info.flags=0x0 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb97b7ee640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-23 to cache LRU end 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb97b7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28): adding source pipe: {s={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb97b7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28): adding dest pipe: {s={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb97b7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb97b7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb97b7ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2773c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2773c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2773c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f2773c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f278960:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2bc2e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f278960:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2e5350:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2e5350:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:05 +0000 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114605Z 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = a96d6e93a1ac9d181c2132566b3f6c20438156b60c65acff556e8d0f44c82d7d 2026-03-20T11:46:05.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:05.270 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114605Z 2026-03-20T11:46:05.270 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:05.270 INFO:teuthology.orchestra.run.vm02.stderr:a96d6e93a1ac9d181c2132566b3f6c20438156b60c65acff556e8d0f44c82d7d 2026-03-20T11:46:05.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:05.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:05.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:05.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:05.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 2a1481b8975aa92ef6eeea1deebc7dca87f4f2b883f4a14102a9c29b9ebf38f7 2026-03-20T11:46:05.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2a1481b8975aa92ef6eeea1deebc7dca87f4f2b883f4a14102a9c29b9ebf38f7 2026-03-20T11:46:05.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:05.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114605Z 2026-03-20T11:46:05.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:05.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 20 register_request mgr=0x7ffecb5f34e0 req_data->id=49, curl_handle=0x55995f28b500 2026-03-20T11:46:05.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb9cb610900 20 run: stack=0x55995f272de0 is io blocked 2026-03-20T11:46:05.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.270+0000 7fb926ffd640 20 link_request req_data=0x7fb95c007fd0 req_data->id=49, curl_handle=0x55995f28b500 2026-03-20T11:46:05.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2bebc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2bebc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:05.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2e1550:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 15 stack 0x55995f2bfaa0 end 2026-03-20T11:46:05.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 run: stack=0x55995f2bfaa0 is done 2026-03-20T11:46:05.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995edfa080:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 collect(): s=0x55995f276250 stack=0x55995f2bfaa0 is complete 2026-03-20T11:46:05.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:05.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f292ef0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:05.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.0 2026-03-20T11:46:05.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.1 2026-03-20T11:46:05.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.2 2026-03-20T11:46:05.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.3 2026-03-20T11:46:05.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.4 2026-03-20T11:46:05.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.5 2026-03-20T11:46:05.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.6 2026-03-20T11:46:05.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.7 2026-03-20T11:46:05.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.8 2026-03-20T11:46:05.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.9 2026-03-20T11:46:05.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.10 2026-03-20T11:46:05.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 run: stack=0x55995f276250 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2c9290:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 cr:s=0x55995f2934e0:op=0x55995f2727d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f2e1550:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 cr:s=0x55995f273020:op=0x55995f2c6970:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2bf450:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 cr:s=0x55995f2c6680:op=0x55995f28b540:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 cr:s=0x55995f2c85f0:op=0x55995f2bebc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 cr:s=0x55995f2e0f60:op=0x55995edfc9c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 cr:s=0x55995edf5d70:op=0x55995f310790:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 cr:s=0x55995f2dbf80:op=0x55995f287070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 cr:s=0x55995f2779b0:op=0x55995f2773c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2c9290:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 run: stack=0x55995f2bfaa0 is io blocked 2026-03-20T11:46:05.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 cr:s=0x55995f2934e0:op=0x55995f2727d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 run: stack=0x55995f2934e0 is io blocked 2026-03-20T11:46:05.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f2e1550:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 run: stack=0x55995f28ec10 is io blocked 2026-03-20T11:46:05.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 cr:s=0x55995f273020:op=0x55995f2c6970:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 run: stack=0x55995f273020 is io blocked 2026-03-20T11:46:05.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2bf450:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 run: stack=0x55995f291350 is io blocked 2026-03-20T11:46:05.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 cr:s=0x55995f2c6680:op=0x55995f28b540:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 run: stack=0x55995f2c6680 is io blocked 2026-03-20T11:46:05.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 cr:s=0x55995f2c85f0:op=0x55995f2bebc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 run: stack=0x55995f2c85f0 is io blocked 2026-03-20T11:46:05.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 cr:s=0x55995f2e0f60:op=0x55995edfc9c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 run: stack=0x55995f2e0f60 is io blocked 2026-03-20T11:46:05.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 cr:s=0x55995edf5d70:op=0x55995f310790:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 run: stack=0x55995edf5d70 is io blocked 2026-03-20T11:46:05.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.281+0000 7fb9cb610900 20 cr:s=0x55995f2dbf80:op=0x55995f287070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.282+0000 7fb9cb610900 20 run: stack=0x55995f2dbf80 is io blocked 2026-03-20T11:46:05.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.282+0000 7fb9cb610900 20 cr:s=0x55995f2779b0:op=0x55995f2773c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.282+0000 7fb9cb610900 20 run: stack=0x55995f2779b0 is io blocked 2026-03-20T11:46:05.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.282+0000 7fb9cb610900 20 cr:s=0x55995f2934e0:op=0x55995f2727d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.282+0000 7fb9cb610900 20 cr:s=0x55995f2934e0:op=0x55995f2727d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.282+0000 7fb9cb610900 20 cr:s=0x55995f2934e0:op=0x55995f2727d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.282+0000 7fb9cb610900 20 cr:s=0x55995f2934e0:op=0x55995f2727d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.282+0000 7fb9cb610900 15 stack 0x55995f2934e0 end 2026-03-20T11:46:05.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.282+0000 7fb9cb610900 20 run: stack=0x55995f2934e0 is done 2026-03-20T11:46:05.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.282+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f292ef0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:05.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.282+0000 7fb9cb610900 20 run: stack=0x55995f276250 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.282+0000 7fb9cb610900 20 cr:s=0x55995f273020:op=0x55995f2c6970:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.282+0000 7fb9cb610900 20 cr:s=0x55995f273020:op=0x55995f2c6970:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.282+0000 7fb9cb610900 20 cr:s=0x55995f273020:op=0x55995f2c6970:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.282+0000 7fb9cb610900 20 cr:s=0x55995f273020:op=0x55995f2c6970:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.282+0000 7fb9cb610900 15 stack 0x55995f273020 end 2026-03-20T11:46:05.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.282+0000 7fb9cb610900 20 run: stack=0x55995f273020 is done 2026-03-20T11:46:05.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.282+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f292ef0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:05.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.282+0000 7fb9cb610900 20 run: stack=0x55995f276250 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2bf450:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2bf450:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995edf5d70:op=0x55995f310790:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2bf450:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995edf5d70:op=0x55995f310790:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f291350:op=0x55995f2bf450:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 15 stack 0x55995f291350 end 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 run: stack=0x55995f291350 is done 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995edf5d70:op=0x55995f310790:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f292ef0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 run: stack=0x55995f276250 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995edf5d70:op=0x55995f310790:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 15 stack 0x55995edf5d70 end 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 run: stack=0x55995edf5d70 is done 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f292ef0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 run: stack=0x55995f276250 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f2e1550:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f2e1550:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f2c85f0:op=0x55995f2bebc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f2e0f60:op=0x55995edfc9c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2c9290:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f2e1550:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f2c85f0:op=0x55995f2bebc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f2e0f60:op=0x55995edfc9c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2c9290:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f28ec10:op=0x55995f2e1550:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 15 stack 0x55995f28ec10 end 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 run: stack=0x55995f28ec10 is done 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f2c85f0:op=0x55995f2bebc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f2e0f60:op=0x55995edfc9c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2c9290:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f292ef0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 run: stack=0x55995f276250 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f2c85f0:op=0x55995f2bebc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 15 stack 0x55995f2c85f0 end 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 run: stack=0x55995f2c85f0 is done 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f2e0f60:op=0x55995edfc9c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 15 stack 0x55995f2e0f60 end 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 run: stack=0x55995f2e0f60 is done 2026-03-20T11:46:05.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f2bfaa0:op=0x55995f2c9290:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 15 stack 0x55995f2bfaa0 end 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 run: stack=0x55995f2bfaa0 is done 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f292ef0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f292ef0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f292ef0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 run: stack=0x55995f276250 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f2c6680:op=0x55995f28b540:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f2c6680:op=0x55995f28b540:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f2c6680:op=0x55995f28b540:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f2c6680:op=0x55995f28b540:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 15 stack 0x55995f2c6680 end 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 run: stack=0x55995f2c6680 is done 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f292ef0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 run: stack=0x55995f276250 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f2779b0:op=0x55995f2773c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f2779b0:op=0x55995f2773c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f2779b0:op=0x55995f2773c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f2779b0:op=0x55995f2773c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 15 stack 0x55995f2779b0 end 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 run: stack=0x55995f2779b0 is done 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f292ef0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.283+0000 7fb9cb610900 20 run: stack=0x55995f276250 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.284+0000 7fb9cb610900 20 cr:s=0x55995f2dbf80:op=0x55995f287070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.284+0000 7fb9cb610900 20 cr:s=0x55995f2dbf80:op=0x55995f287070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.284+0000 7fb9cb610900 20 cr:s=0x55995f2dbf80:op=0x55995f287070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.284+0000 7fb9cb610900 20 cr:s=0x55995f2dbf80:op=0x55995f287070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.284+0000 7fb9cb610900 15 stack 0x55995f2dbf80 end 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.284+0000 7fb9cb610900 20 run: stack=0x55995f2dbf80 is done 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.284+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f292ef0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.284+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995edfa080:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.284+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2727d0:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.284+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2727d0:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.284+0000 7fb9cb610900 20 enqueued request req=0x55995f2750c0 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.284+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.284+0000 7fb9cb610900 20 req: 0x55995f2750c0 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.284+0000 7fb9cb610900 20 run: stack=0x55995f276250 is io blocked 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.284+0000 7fb97afed640 20 dequeued request req=0x55995f2750c0 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.284+0000 7fb97afed640 20 RGWWQ: empty 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.285+0000 7fb97afed640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 info.flags=0x11 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.285+0000 7fb97afed640 10 rgw async rados processor: moving a2.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 to cache LRU end 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.285+0000 7fb97afed640 10 rgw async rados processor: distributing notification oid=a2.rgw.control:notify.5 cni=[op: 0, obj: a2.rgw.meta:root:.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3, ofs0, ns] 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.285+0000 7fb9a2ffd640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 343597383760 cookie 94117215055120 notifier 6376 bl.length()=599 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.285+0000 7fb9a2ffd640 10 rgw watcher librados: cache put: name=a2.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 info.flags=0x11 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.285+0000 7fb9a2ffd640 10 rgw watcher librados: moving a2.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 to cache LRU end 2026-03-20T11:46:05.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.286+0000 7fb97afed640 20 rgw async rados processor: handle_bi_update(): bucket=testx:tenanted-bucket[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3]): orig_sources= new_sources= 2026-03-20T11:46:05.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.286+0000 7fb97afed640 20 rgw async rados processor: handle_bi_update(): bucket=testx:tenanted-bucket[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3]): potential sources added=[] removed=[] 2026-03-20T11:46:05.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.286+0000 7fb97afed640 20 rgw async rados processor: handle_bi_update(): bucket=testx:tenanted-bucket[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3]): orig_dests= new_dests= 2026-03-20T11:46:05.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.286+0000 7fb97afed640 20 rgw async rados processor: handle_bi_update(): bucket=testx:tenanted-bucket[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3]): potential dests added=[] removed=[] 2026-03-20T11:46:05.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.286+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2727d0:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:05.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.286+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2727d0:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:05.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.286+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2727d0:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:05.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.286+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2727d0:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:05.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.286+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995edfa080:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.286+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2773c0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:05.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.286+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2773c0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:05.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.286+0000 7fb9cb610900 20 enqueued request req=0x55995ed93f40 2026-03-20T11:46:05.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.286+0000 7fb9cb610900 20 RGWWQ: 2026-03-20T11:46:05.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.286+0000 7fb9cb610900 20 req: 0x55995ed93f40 2026-03-20T11:46:05.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.286+0000 7fb9cb610900 20 run: stack=0x55995f276250 is io blocked 2026-03-20T11:46:05.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.286+0000 7fb97a7ec640 20 dequeued request req=0x55995ed93f40 2026-03-20T11:46:05.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.286+0000 7fb97a7ec640 20 RGWWQ: empty 2026-03-20T11:46:05.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.286+0000 7fb97a7ec640 10 rgw async rados processor: removing a2.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 from cache 2026-03-20T11:46:05.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.286+0000 7fb97a7ec640 10 rgw async rados processor: distributing notification oid=a2.rgw.control:notify.5 cni=[op: 1, obj: a2.rgw.meta:root:.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3, ofs0, ns] 2026-03-20T11:46:05.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.287+0000 7fb9a2ffd640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 343597383761 cookie 94117215055120 notifier 6376 bl.length()=217 2026-03-20T11:46:05.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.287+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2773c0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:05.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.287+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2773c0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:05.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.287+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2773c0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:05.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.287+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995f2773c0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:05.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.287+0000 7fb9cb610900 20 cr:s=0x55995f276250:op=0x55995edfa080:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.287+0000 7fb9cb610900 20 trim: trimmed bucket instance testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 2026-03-20T11:46:05.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.287+0000 7fb9cb610900 15 stack 0x55995f276250 end 2026-03-20T11:46:05.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.287+0000 7fb9cb610900 20 run: stack=0x55995f276250 is done 2026-03-20T11:46:05.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.287+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:05.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.287+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.310 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.313+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2e5350:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.310 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.313+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2e5350:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.310 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.313+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2e5350:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.310 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.313+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2e5350:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:05.310 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.313+0000 7fb9cb610900 20 cr:s=0x55995f272de0:op=0x55995f2bc2e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:05.310 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.313+0000 7fb9cb610900 15 stack 0x55995f272de0 end 2026-03-20T11:46:05.310 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.313+0000 7fb9cb610900 20 run: stack=0x55995f272de0 is done 2026-03-20T11:46:05.310 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.313+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f278960:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.310 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.313+0000 7fb9cb610900 20 collect(): s=0x55995f18b1a0 stack=0x55995f272de0 is complete 2026-03-20T11:46:05.310 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.313+0000 7fb9cb610900 10 min_generation is 0 2026-03-20T11:46:05.310 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.313+0000 7fb9cb610900 10 trimming bilogs for bucket=:dipepu-23[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28]) markers=[,,,,,,,,00000000002.27.13,,], shards=11 2026-03-20T11:46:05.310 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.313+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f292ef0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.310 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.313+0000 7fb9cb610900 10 trimming bilog shard 8 of :dipepu-23[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28]) at marker 00000000002.27.13 2026-03-20T11:46:05.310 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.313+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:05.310 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.313+0000 7fb9cb610900 20 cr:s=0x55995f2c85f0:op=0x55995f2d84a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.310 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.313+0000 7fb9cb610900 20 cr:s=0x55995f2c85f0:op=0x55995f2d84a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.310 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.313+0000 7fb9cb610900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28.8 2026-03-20T11:46:05.310 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.313+0000 7fb9cb610900 20 run: stack=0x55995f2c85f0 is io blocked 2026-03-20T11:46:05.311 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.314+0000 7fb9cb610900 20 cr:s=0x55995f2c85f0:op=0x55995f2d84a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.311 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.314+0000 7fb9cb610900 20 cr:s=0x55995f2c85f0:op=0x55995f2d84a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.311 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.314+0000 7fb9cb610900 20 cr:s=0x55995f2c85f0:op=0x55995f2d84a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.311 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.314+0000 7fb9cb610900 20 cr:s=0x55995f2c85f0:op=0x55995f2d84a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:05.311 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.314+0000 7fb9cb610900 15 stack 0x55995f2c85f0 end 2026-03-20T11:46:05.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.314+0000 7fb9cb610900 20 run: stack=0x55995f2c85f0 is done 2026-03-20T11:46:05.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.314+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f292ef0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:05.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.314+0000 7fb9cb610900 20 cr:s=0x55995f18b1a0:op=0x55995f278960:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:05.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.314+0000 7fb9cb610900 20 trim: trimmed bucket instance dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:05.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.314+0000 7fb9cb610900 15 stack 0x55995f18b1a0 end 2026-03-20T11:46:05.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.314+0000 7fb9cb610900 20 run: stack=0x55995f18b1a0 is done 2026-03-20T11:46:05.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.314+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:05.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.314+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f274610:27BucketTrimInstanceCollectCR: operate() returned r=-22 2026-03-20T11:46:05.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.314+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f270290:12BucketTrimCR: operate() 2026-03-20T11:46:05.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.314+0000 7fb9cb610900 20 writing bucket trim marker=16:c59acbbb:root::dipepu-27:head 2026-03-20T11:46:05.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.314+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f2d84a0:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:05.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.314+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f2d84a0:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:05.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.315+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is io blocked 2026-03-20T11:46:05.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.316+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f2d84a0:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:05.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.316+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f2d84a0:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:05.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.316+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f2d84a0:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:05.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.316+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f2d84a0:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:05.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.316+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f270290:12BucketTrimCR: operate() 2026-03-20T11:46:05.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.316+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f2d84a0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:05.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.316+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f2d84a0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:05.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.316+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is io blocked 2026-03-20T11:46:05.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.316+0000 7fb9a2ffd640 20 trim: bucket trim completed 2026-03-20T11:46:05.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.317+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f2d84a0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:05.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.317+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f2d84a0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:05.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.317+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f2d84a0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:05.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.317+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f2d84a0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:05.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.317+0000 7fb9cb610900 20 cr:s=0x55995f271b90:op=0x55995f270290:12BucketTrimCR: operate() 2026-03-20T11:46:05.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.317+0000 7fb9cb610900 4 bucket index log processing completed in 0.774279475s 2026-03-20T11:46:05.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.317+0000 7fb9cb610900 15 stack 0x55995f271b90 end 2026-03-20T11:46:05.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.317+0000 7fb9cb610900 20 run: stack=0x55995f271b90 is done 2026-03-20T11:46:05.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.324+0000 7fb9cb610900 20 remove_watcher() i=4 2026-03-20T11:46:05.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.324+0000 7fb9cb610900 2 removed watcher, disabling cache 2026-03-20T11:46:05.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.324+0000 7fb9cb610900 20 remove_watcher() i=1 2026-03-20T11:46:05.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.325+0000 7fb9cb610900 20 remove_watcher() i=3 2026-03-20T11:46:05.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.325+0000 7fb9cb610900 20 remove_watcher() i=5 2026-03-20T11:46:05.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.325+0000 7fb9cb610900 20 remove_watcher() i=2 2026-03-20T11:46:05.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.325+0000 7fb9cb610900 20 remove_watcher() i=0 2026-03-20T11:46:05.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.325+0000 7fb9cb610900 20 remove_watcher() i=7 2026-03-20T11:46:05.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:05.325+0000 7fb9cb610900 20 remove_watcher() i=6 2026-03-20T11:46:10.373 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:10.374 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:10.374 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:46:10.410 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:10.410 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:10.501 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:46:10.512 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774007137.857782_141.1","last_update":"2026-03-20T11:45:37.857782Z"},{"marker":"1_1774007137.912792_174.1","last_update":"2026-03-20T11:45:37.912792Z"},{"marker":"1_1774007157.850890_269.1","last_update":"2026-03-20T11:45:57.850890Z"},{"marker":"1_1774007039.336409_210.1","last_update":"2026-03-20T11:43:59.336409Z"}] 2026-03-20T11:46:10.512 INFO:rgw_multi.tests:starting meta checkpoint for zone=b1 2026-03-20T11:46:10.512 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:10.512 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:10.512 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:46:10.553 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:10.553 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:10.632 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.634+0000 7f11c6552900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:46:10.633 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.635+0000 7f11c6552900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:46:10.643 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774007137.857782_141.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:45:37.857782Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774007137.912792_174.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:37.912792Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774007157.850890_269.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:57.850890Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774007039.336409_210.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:59.336409Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:46:10Z"} 2026-03-20T11:46:10.644 INFO:rgw_multi.tests:finish meta checkpoint for zone=b1 2026-03-20T11:46:10.644 INFO:rgw_multi.tests:starting meta checkpoint for zone=b2 2026-03-20T11:46:10.644 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:10.644 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:10.644 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b2 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:46:10.723 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:10.723 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:10.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.788+0000 7f88b3020900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:46:10.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.789+0000 7f88b3020900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:46:10.797 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774007137.857782_141.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:45:37.857782Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774007137.912792_174.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:37.912792Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774007157.850890_269.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:57.850890Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774007039.336409_210.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:59.336409Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:46:10Z"} 2026-03-20T11:46:10.797 INFO:rgw_multi.tests:finish meta checkpoint for zone=b2 2026-03-20T11:46:10.798 INFO:rgw_multi.tests:trimming on zone=b1 2026-03-20T11:46:10.798 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '50', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:46:10.798 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '50', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:46:10.798 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bilog autotrim --rgw-sync-log-trim-max-buckets 50 --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 20 --debug-ms 0 2026-03-20T11:46:10.875 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:10.875 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:10.894 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.896+0000 7f7363925900 20 rados->read ofs=0 len=0 2026-03-20T11:46:10.898 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.899+0000 7f7363925900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:46:10.898 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.899+0000 7f7363925900 20 rados->read ofs=0 len=0 2026-03-20T11:46:10.898 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.900+0000 7f7363925900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:46:10.898 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.900+0000 7f7363925900 20 realm test-realm 6a789b89-c2b8-45f5-82cc-fb80234d0c6c 2026-03-20T11:46:10.898 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.900+0000 7f7363925900 20 rados->read ofs=0 len=0 2026-03-20T11:46:10.898 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.900+0000 7f7363925900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:46:10.898 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.900+0000 7f7363925900 20 rados->read ofs=0 len=0 2026-03-20T11:46:10.899 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.901+0000 7f7363925900 20 rados_obj.operate() r=0 bl.length=10 2026-03-20T11:46:10.899 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.901+0000 7f7363925900 20 rados->read ofs=0 len=0 2026-03-20T11:46:10.899 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.901+0000 7f7363925900 20 rados_obj.operate() r=0 bl.length=2020 2026-03-20T11:46:10.899 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.901+0000 7f7363925900 20 rados->read ofs=0 len=0 2026-03-20T11:46:10.900 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.902+0000 7f7363925900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:46:10.900 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.902+0000 7f7363925900 20 rados->read ofs=0 len=0 2026-03-20T11:46:10.900 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.902+0000 7f7363925900 20 rados_obj.operate() r=0 bl.length=1030 2026-03-20T11:46:10.900 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.902+0000 7f7363925900 20 zone b1 found 2026-03-20T11:46:10.900 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.902+0000 7f7363925900 4 Realm: test-realm (6a789b89-c2b8-45f5-82cc-fb80234d0c6c) 2026-03-20T11:46:10.900 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.902+0000 7f7363925900 4 ZoneGroup: b (b) 2026-03-20T11:46:10.900 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.902+0000 7f7363925900 4 Zone: b1 (efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c) 2026-03-20T11:46:10.900 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.902+0000 7f7363925900 4 using period configuration: cb2fcdce-2c1b-429a-8601-a284ee3a2381:12 2026-03-20T11:46:10.900 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.902+0000 7f7363925900 20 period zonegroup name b 2026-03-20T11:46:10.900 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.902+0000 7f7363925900 20 using current period zonegroup b 2026-03-20T11:46:10.900 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.902+0000 7f7363925900 20 rados->read ofs=0 len=0 2026-03-20T11:46:10.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.903+0000 7f7363925900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:10.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.903+0000 7f7363925900 20 rados->read ofs=0 len=0 2026-03-20T11:46:10.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.903+0000 7f7363925900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:10.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.903+0000 7f7363925900 20 init(): pipe_map (zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:46:10.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.903+0000 7f7363925900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:10.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.903+0000 7f7363925900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:10.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.903+0000 7f7363925900 20 rados->read ofs=0 len=0 2026-03-20T11:46:10.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.903+0000 7f7363925900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:10.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.903+0000 7f7363925900 20 rados->read ofs=0 len=0 2026-03-20T11:46:10.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.903+0000 7f7363925900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:10.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.903+0000 7f7363925900 20 init(): pipe_map (zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:46:10.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.903+0000 7f7363925900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:10.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.903+0000 7f7363925900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:10.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.903+0000 7f7363925900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:10.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.903+0000 7f7363925900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:10.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.903+0000 7f7363925900 20 started sync module instance, tier type = 2026-03-20T11:46:10.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.903+0000 7f7363925900 20 generating connection object for zone b2 id cd52b229-0d4a-43bf-b4ba-156aa19e7aef 2026-03-20T11:46:10.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.903+0000 7f7363925900 20 started zone id=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c (name=b1) with tier type = 2026-03-20T11:46:10.906 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.908+0000 7f7363925900 20 add_watcher() i=0 2026-03-20T11:46:10.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.911+0000 7f7363925900 20 add_watcher() i=5 2026-03-20T11:46:10.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.911+0000 7f7363925900 20 add_watcher() i=6 2026-03-20T11:46:10.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.915+0000 7f7363925900 20 add_watcher() i=4 2026-03-20T11:46:10.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.915+0000 7f7363925900 20 add_watcher() i=7 2026-03-20T11:46:10.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.915+0000 7f7363925900 20 add_watcher() i=2 2026-03-20T11:46:10.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.915+0000 7f7363925900 20 add_watcher() i=1 2026-03-20T11:46:10.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.915+0000 7f7363925900 20 add_watcher() i=3 2026-03-20T11:46:10.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.915+0000 7f7363925900 2 all 8 watchers are set, enabling cache 2026-03-20T11:46:10.917 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.919+0000 7f7363925900 20 rgw_check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2026-03-20T11:46:10.917 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.919+0000 7f7363925900 20 rgw_check_secure_mon_conn(): mode 1 is insecure 2026-03-20T11:46:10.917 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.919+0000 7f7363925900 5 note: GC not initialized 2026-03-20T11:46:10.970 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.972+0000 7f7363925900 20 init_complete bucket index max shards: 11 2026-03-20T11:46:10.970 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.972+0000 7f7363925900 20 Filter name: none 2026-03-20T11:46:10.970 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.972+0000 7f72bffff640 20 reqs_thread_entry: start 2026-03-20T11:46:10.970 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.972+0000 7f72be7fc640 20 reqs_thread_entry: start 2026-03-20T11:46:10.971 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.973+0000 7f7363925900 10 rgw bucket trim manager: Watching bilog.trim 2026-03-20T11:46:10.971 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.973+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c2276749c0:12BucketTrimCR: operate() 2026-03-20T11:46:10.971 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.973+0000 7f7363925900 10 fetching active bucket counters 2026-03-20T11:46:10.971 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.973+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c2276770e0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:10.971 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.973+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c2276770e0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:10.971 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.973+0000 7f7363925900 20 run: stack=0x55c227674780 is io blocked 2026-03-20T11:46:10.972 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.974+0000 7f733a7fc640 20 trim: get_bucket_counters: [] 2026-03-20T11:46:10.972 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.974+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c2276770e0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:10.972 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.974+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c2276770e0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:10.972 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.974+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c2276770e0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:10.972 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.974+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c2276770e0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:10.972 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.974+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c2276749c0:12BucketTrimCR: operate() 2026-03-20T11:46:10.972 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.974+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:10.972 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.974+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:10.972 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.974+0000 7f7363925900 20 run: stack=0x55c227674780 is io blocked 2026-03-20T11:46:10.973 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.974+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:10.973 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.974+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:10.973 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.974+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:10.973 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.974+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:10.973 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.974+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c2276749c0:12BucketTrimCR: operate() 2026-03-20T11:46:10.973 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.974+0000 7f7363925900 10 listing cold buckets from marker= 2026-03-20T11:46:10.973 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.974+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:14MetadataListCR: operate() 2026-03-20T11:46:10.973 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.974+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:14MetadataListCR: operate() 2026-03-20T11:46:10.973 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.974+0000 7f7363925900 20 enqueued request req=0x7f731c002970 2026-03-20T11:46:10.973 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.974+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:10.973 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.974+0000 7f7363925900 20 req: 0x7f731c002970 2026-03-20T11:46:10.973 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.974+0000 7f7363925900 20 run: stack=0x55c227674780 is io blocked 2026-03-20T11:46:10.973 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.975+0000 7f730e7e4640 20 dequeued request req=0x7f731c002970 2026-03-20T11:46:10.973 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.975+0000 7f730e7e4640 20 RGWWQ: empty 2026-03-20T11:46:10.973 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.975+0000 7f730e7e4640 20 rgw async rados processor: starting metadata listing at 2026-03-20T11:46:10.980 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.982+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-10 2026-03-20T11:46:10.980 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.982+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:10.987 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.989+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:10.993 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:10.995+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:11.000 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.002+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:11.006 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.008+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:11.011 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.013+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:11.017 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.019+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-19 2026-03-20T11:46:11.017 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.019+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-3 2026-03-20T11:46:11.017 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.019+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:11.023 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.025+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:11.029 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.031+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-17 2026-03-20T11:46:11.029 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.031+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:11.035 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.037+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-38 2026-03-20T11:46:11.035 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.037+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:11.041 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.043+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-24 2026-03-20T11:46:11.041 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.043+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-36 2026-03-20T11:46:11.041 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.043+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:11.046 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.048+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:11.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.054+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:11.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.058+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-44 2026-03-20T11:46:11.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.058+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-35 2026-03-20T11:46:11.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.058+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:11.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.063+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-16 2026-03-20T11:46:11.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.063+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:11.066 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.068+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:11.071 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.073+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 2026-03-20T11:46:11.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.078+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:11.080 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.082+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-55 2026-03-20T11:46:11.080 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.082+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-13 2026-03-20T11:46:11.081 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.082+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-48 2026-03-20T11:46:11.081 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.082+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-25 2026-03-20T11:46:11.081 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.082+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-51 2026-03-20T11:46:11.081 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.082+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:11.085 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.087+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:11.089 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.091+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-46 2026-03-20T11:46:11.089 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.091+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:46:11.093 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.095+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:46:11.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.099+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:46:11.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.103+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-26 2026-03-20T11:46:11.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.103+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-20 2026-03-20T11:46:11.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.103+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:46:11.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.107+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:46:11.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.110+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-1 2026-03-20T11:46:11.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.110+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:46:11.113 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.115+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:46:11.116 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.118+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:46:11.120 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.122+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-52 2026-03-20T11:46:11.120 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.122+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-32 2026-03-20T11:46:11.120 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.122+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-42 2026-03-20T11:46:11.120 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.122+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:46:11.123 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.125+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:46:11.126 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.128+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-7 2026-03-20T11:46:11.126 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.128+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-54 2026-03-20T11:46:11.126 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.128+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-49 2026-03-20T11:46:11.126 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.128+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-47 2026-03-20T11:46:11.127 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.128+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 2026-03-20T11:46:11.130 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.132+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:46:11.133 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.135+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-11 2026-03-20T11:46:11.133 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.135+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-12 2026-03-20T11:46:11.133 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.135+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-41 2026-03-20T11:46:11.133 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.135+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-23 2026-03-20T11:46:11.133 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.135+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-15 2026-03-20T11:46:11.133 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.135+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-45 2026-03-20T11:46:11.133 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.135+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:46:11.137 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.139+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:46:11.139 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.141+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-9 2026-03-20T11:46:11.139 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.141+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:46:11.142 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.144+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-29 2026-03-20T11:46:11.142 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.144+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:46:11.145 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.147+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-31 2026-03-20T11:46:11.145 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.147+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:46:11.147 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.149+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-33 2026-03-20T11:46:11.147 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.149+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:11.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.152+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-4 2026-03-20T11:46:11.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.152+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:46:11.152 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.154+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:11.154 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.156+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:11.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.158+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-43 2026-03-20T11:46:11.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.158+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-30 2026-03-20T11:46:11.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.158+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-18 2026-03-20T11:46:11.157 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.158+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-2 2026-03-20T11:46:11.157 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.158+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:46:11.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.160+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-8 2026-03-20T11:46:11.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.160+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:46:11.160 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.162+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:11.162 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.164+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-58 2026-03-20T11:46:11.162 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.164+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:46:11.163 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.165+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 2026-03-20T11:46:11.165 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.167+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-27 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-28 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-59 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f730e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:14MetadataListCR: operate() 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:14MetadataListCR: operate() 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:14MetadataListCR: operate() 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:14MetadataListCR: operate() 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c2276749c0:12BucketTrimCR: operate() 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 4 collected 50 buckets for trim 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 4 starting trim on bucket=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 cr:s=0x55c22758e610:op=0x55c227679aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 4 starting trim on bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 cr:s=0x55c227678db0:op=0x55c22767ad70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 4 starting trim on bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22767c040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 4 starting trim on bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22767d310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 cr:s=0x55c22758e610:op=0x55c22767da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 cr:s=0x55c227678db0:op=0x55c22767e1b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22767ebd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22767d310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 enqueued request req=0x55c22726f640 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 req: 0x55c22726f640 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is io blocked 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 cr:s=0x55c22758e610:op=0x55c22767da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 enqueued request req=0x55c22726f9e0 2026-03-20T11:46:11.167 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.168 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 req: 0x55c22726f640 2026-03-20T11:46:11.168 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 req: 0x55c22726f9e0 2026-03-20T11:46:11.173 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.173+0000 7f3e2c03e640 -1 req 17942674297608984451 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.178 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 run: stack=0x55c22758e610 is io blocked 2026-03-20T11:46:11.178 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 cr:s=0x55c227678db0:op=0x55c22767e1b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.178 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 enqueued request req=0x55c22726fe00 2026-03-20T11:46:11.178 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.178 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 req: 0x55c22726f640 2026-03-20T11:46:11.178 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 req: 0x55c22726f9e0 2026-03-20T11:46:11.178 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 req: 0x55c22726fe00 2026-03-20T11:46:11.178 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 run: stack=0x55c227678db0 is io blocked 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22767ebd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 enqueued request req=0x55c227270220 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 req: 0x55c22726f640 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 req: 0x55c22726f9e0 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 req: 0x55c22726fe00 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 req: 0x55c227270220 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7363925900 20 run: stack=0x55c227677bf0 is io blocked 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f73267fc640 20 dequeued request req=0x55c22726f640 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f73267fc640 20 RGWWQ: 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f73267fc640 20 req: 0x55c22726f9e0 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f73267fc640 20 req: 0x55c22726fe00 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f73267fc640 20 req: 0x55c227270220 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f73267fc640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 : miss 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f73267fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f731bfff640 20 dequeued request req=0x55c22726f9e0 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f731bfff640 20 RGWWQ: 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f731bfff640 20 req: 0x55c22726fe00 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f731bfff640 20 req: 0x55c227270220 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f731bfff640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 : miss 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f731bfff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7324ff9640 20 dequeued request req=0x55c22726fe00 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7324ff9640 20 RGWWQ: 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7324ff9640 20 req: 0x55c227270220 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7324ff9640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 : miss 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f7324ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f73177f6640 20 dequeued request req=0x55c227270220 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f73177f6640 20 RGWWQ: empty 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f73177f6640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 : miss 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.168+0000 7f73177f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f73177f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f73177f6640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 info.flags=0x17 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f73177f6640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 to cache LRU end 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f73177f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f73177f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f73177f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f73177f6640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-55 : miss 2026-03-20T11:46:11.179 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f73177f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f7324ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f7324ff9640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 info.flags=0x17 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f7324ff9640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 to cache LRU end 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f7324ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f7324ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f7324ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f7324ff9640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-54 : miss 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f7324ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f73267fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=387 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f73267fc640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 info.flags=0x17 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f73267fc640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 to cache LRU end 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f73267fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f73267fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f73267fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f73267fc640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-60 : miss 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f73267fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f731bfff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f731bfff640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 info.flags=0x17 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f731bfff640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 to cache LRU end 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f731bfff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f731bfff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f731bfff640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f731bfff640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-53 : miss 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f731bfff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f73177f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f73177f6640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-55 info.flags=0x0 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f73177f6640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-55 to cache LRU end 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f73177f6640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-55 : miss 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f73177f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f7324ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f73267fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f7324ff9640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-54 info.flags=0x0 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f7324ff9640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-54 to cache LRU end 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f73267fc640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-60 info.flags=0x0 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f73267fc640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-60 to cache LRU end 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f73267fc640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-60 : miss 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f73267fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f7324ff9640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-54 : miss 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.169+0000 7f7324ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f731bfff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f731bfff640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-53 info.flags=0x0 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f731bfff640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-53 to cache LRU end 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f731bfff640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-53 : miss 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f731bfff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f731bfff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f731bfff640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-53 info.flags=0x0 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f731bfff640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-53 to cache LRU end 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f731bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13): adding source pipe: {s={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f731bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13): adding dest pipe: {s={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.180 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f731bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f731bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f731bfff640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f73267fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f73267fc640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-60 info.flags=0x0 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f73267fc640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-60 to cache LRU end 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f73267fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20): adding source pipe: {s={b=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f73267fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20): adding dest pipe: {s={b=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f73267fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f73267fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f73267fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c22758e610:op=0x55c22767da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c22758e610:op=0x55c22767da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22767d310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c22758e610:op=0x55c22767da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22767d310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c22758e610:op=0x55c22767da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22767d310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c22758e610:op=0x55c227679aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22767d310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c227677670:op=0x55c227270640:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c22758e610:op=0x55c227679aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 run: stack=0x55c22758e610 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c227677670:op=0x55c22767d310:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227270c60:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c227677670:op=0x55c22767d310:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = e9126c939be667bf5c0e1f6f613d5d5b12c5350ff0a0f98b006fad35520bb225 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:e9126c939be667bf5c0e1f6f613d5d5b12c5350ff0a0f98b006fad35520bb225 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = c0ec6fad09f2eefc04161c105182d15d252d197fbeff0ee6d76e9a55914ad809 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c0ec6fad09f2eefc04161c105182d15d252d197fbeff0ee6d76e9a55914ad809 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=0, curl_handle=0x55c227273060 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 run: stack=0x55c227677670 is io blocked 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227271280:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227271280:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f72be7fc640 20 link_request req_data=0x55c22767dd80 req_data->id=0, curl_handle=0x55c227273060 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 354e12d340172b3b611f7ce73434e01db611b696f00faa03779fa0883c30d0cc 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:354e12d340172b3b611f7ce73434e01db611b696f00faa03779fa0883c30d0cc 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.182 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = a74f6576ed40226e7bc3cf48815b2ef844dae38498d0b64eea04c597eb4d7631 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a74f6576ed40226e7bc3cf48815b2ef844dae38498d0b64eea04c597eb4d7631 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=1, curl_handle=0x55c2272606a0 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 run: stack=0x55c22767e040 is io blocked 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f73177f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f73177f6640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-55 info.flags=0x0 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f73177f6640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-55 to cache LRU end 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f72be7fc640 20 link_request req_data=0x55c227260da0 req_data->id=1, curl_handle=0x55c2272606a0 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f73177f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding source pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f73177f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding dest pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f73177f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f73177f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f73177f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7324ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7324ff9640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-54 info.flags=0x0 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7324ff9640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-54 to cache LRU end 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7324ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14): adding source pipe: {s={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7324ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14): adding dest pipe: {s={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7324ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7324ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7324ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22767ebd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22767ebd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22767ebd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22767ebd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22767c040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c227678db0:op=0x55c22767e1b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c227262db0:op=0x55c227262790:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22767c040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 run: stack=0x55c227677bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c227678db0:op=0x55c22767e1b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c227262db0:op=0x55c227262f40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c227678db0:op=0x55c22767e1b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c227262db0:op=0x55c227262f40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 2b2c04556578e8a29f87627197dd3692e548c00bd0fac4d25faf57767aa95163 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:2b2c04556578e8a29f87627197dd3692e548c00bd0fac4d25faf57767aa95163 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = acdb34da071a83e3d4dc2e5a1c4ae46b8a534c4c276b3e80b846ab63e16ef0a6 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=acdb34da071a83e3d4dc2e5a1c4ae46b8a534c4c276b3e80b846ab63e16ef0a6 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=2, curl_handle=0x55c227264b20 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 run: stack=0x55c227262db0 is io blocked 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c227678db0:op=0x55c22767e1b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c227678db0:op=0x55c22767ad70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c2272656e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f72be7fc640 20 link_request req_data=0x55c227265420 req_data->id=2, curl_handle=0x55c227264b20 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c227678db0:op=0x55c22767ad70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 run: stack=0x55c227678db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c227265d00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c227265d00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.170+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.171+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.171+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.171+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.184 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.171+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.171+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.171+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 8005f9c8252cef86fd794f77adaf02aaf41497a022a6f0de818b5700c9a3b366 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.171+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:8005f9c8252cef86fd794f77adaf02aaf41497a022a6f0de818b5700c9a3b366 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.171+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.171+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.171+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.171+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.171+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 221a095bfe89900b354adbeac64dac7b3d8423baf89764ef19f91e3a0bdb89a0 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.171+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=221a095bfe89900b354adbeac64dac7b3d8423baf89764ef19f91e3a0bdb89a0 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.171+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.171+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.171+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.171+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=3, curl_handle=0x55c227681f00 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.171+0000 7f7363925900 20 run: stack=0x55c22767e740 is io blocked 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.171+0000 7f72be7fc640 20 link_request req_data=0x55c227682650 req_data->id=3, curl_handle=0x55c227681f00 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227271280:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227271280:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227271280:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227271280:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227270c60:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 15 stack 0x55c22767e040 end 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 run: stack=0x55c22767e040 is done 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 collect(): s=0x55c22753d3c0 stack=0x55c22767e040 is complete 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 10 min_generation is 1 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22725f210:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.0 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.2 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.3 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.4 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.5 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.6 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.7 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.8 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.9 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.10 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227270c60:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c2276864c0:op=0x55c227271250:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c227683f70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c227684560:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c22725ffc0:op=0x55c227684b50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c2276891e0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c22768a1c0:op=0x55c2276897d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768a4f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c22768bc00:op=0x55c22768b210:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c22768c920:op=0x55c22768bf30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c22768d640:op=0x55c22768cc50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227270c60:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 run: stack=0x55c22767e040 is io blocked 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c2276864c0:op=0x55c227271250:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 run: stack=0x55c2276864c0 is io blocked 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c227683f70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 run: stack=0x55c227260bc0 is io blocked 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c227684560:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 run: stack=0x55c227687d10 is io blocked 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c22725ffc0:op=0x55c227684b50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 run: stack=0x55c22725ffc0 is io blocked 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c2276891e0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 run: stack=0x55c227685830 is io blocked 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c22768a1c0:op=0x55c2276897d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 run: stack=0x55c22768a1c0 is io blocked 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768a4f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 run: stack=0x55c22768aee0 is io blocked 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c22768bc00:op=0x55c22768b210:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 run: stack=0x55c22768bc00 is io blocked 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c22768c920:op=0x55c22768bf30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 run: stack=0x55c22768c920 is io blocked 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 cr:s=0x55c22768d640:op=0x55c22768cc50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.173+0000 7f7363925900 20 run: stack=0x55c22768d640 is io blocked 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7363925900 20 cr:s=0x55c227262db0:op=0x55c227262f40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7363925900 20 cr:s=0x55c227262db0:op=0x55c227262f40:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7363925900 20 cr:s=0x55c227262db0:op=0x55c227262790:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7363925900 15 stack 0x55c227262db0 end 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7363925900 20 run: stack=0x55c227262db0 is done 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22767c040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7363925900 20 collect(): s=0x55c227677bf0 stack=0x55c227262db0 is complete 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22767c040:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7363925900 15 stack 0x55c227677bf0 end 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7363925900 20 run: stack=0x55c227677bf0 is done 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22767c040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7363925900 4 starting trim on bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7363925900 20 enqueued request req=0x55c227273590 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7363925900 20 req: 0x55c227273590 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7363925900 20 run: stack=0x55c227677bf0 is io blocked 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7325ffb640 20 dequeued request req=0x55c227273590 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7325ffb640 20 RGWWQ: empty 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7325ffb640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 : miss 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.174+0000 7f7325ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.175+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227270c60:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.175+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227270c60:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.175+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227270c60:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.175+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227270c60:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.175+0000 7f7363925900 15 stack 0x55c22767e040 end 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.175+0000 7f7363925900 20 run: stack=0x55c22767e040 is done 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.175+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22725f210:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.175+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.175+0000 7f7325ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.175+0000 7f7325ffb640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 info.flags=0x17 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.175+0000 7f7325ffb640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 to cache LRU end 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.175+0000 7f7325ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.175+0000 7f7325ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.175+0000 7f7325ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.175+0000 7f7325ffb640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-28 : miss 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.175+0000 7f7325ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c227684560:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c227684560:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c227684560:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c227684560:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 15 stack 0x55c227687d10 end 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 run: stack=0x55c227687d10 is done 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22725f210:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c2276891e0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c2276891e0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c2276891e0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c2276891e0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 15 stack 0x55c227685830 end 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 run: stack=0x55c227685830 is done 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22725f210:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7325ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7325ffb640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-28 info.flags=0x0 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7325ffb640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-28 to cache LRU end 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7325ffb640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-28 : miss 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7325ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c22768bc00:op=0x55c22768b210:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.187 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c22768bc00:op=0x55c22768b210:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.187 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.181+0000 7f3e40066640 -1 req 3119036218742134523 0.004000006s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.187 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.184+0000 7f3ea212a640 -1 req 5739544881160377894 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.187 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.187+0000 7f3dfcfe0640 -1 req 17098825691985923492 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.187 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.188+0000 7f3e5b09c640 -1 req 3560600120798950895 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c22768bc00:op=0x55c22768b210:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c22768bc00:op=0x55c22768b210:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 15 stack 0x55c22768bc00 end 2026-03-20T11:46:11.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 run: stack=0x55c22768bc00 is done 2026-03-20T11:46:11.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22725f210:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:11.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c22768a1c0:op=0x55c2276897d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c22768a1c0:op=0x55c2276897d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c227265d00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7325ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7325ffb640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-28 info.flags=0x0 2026-03-20T11:46:11.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7325ffb640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-28 to cache LRU end 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7325ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding source pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7325ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding dest pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7325ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7325ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7325ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c22768a1c0:op=0x55c2276897d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c227265d00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c22768a1c0:op=0x55c2276897d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 15 stack 0x55c22768a1c0 end 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 run: stack=0x55c22768a1c0 is done 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c227265d00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22725f210:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c227265d00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c2272656e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 15 stack 0x55c22767e740 end 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 run: stack=0x55c22767e740 is done 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c227678db0:op=0x55c22767ad70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 collect(): s=0x55c227678db0 stack=0x55c22767e740 is complete 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 10 trimming bilogs for bucket=:dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) markers=[00000000002.35.1,00000000002.30.1,00000000002.46.1,00000000002.27.1,00000000002.25.1,00000000002.47.1,00000000002.30.1,00000000002.31.1,00000000002.40.1,00000000002.59.1,00000000002.24.1], shards=11 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22767c040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c227678db0:op=0x55c227270c60:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 10 trimming bilog shard 0 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.35.1 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 10 trimming bilog shard 1 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.30.1 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 10 trimming bilog shard 2 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.46.1 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 10 trimming bilog shard 3 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.27.1 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 10 trimming bilog shard 4 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.25.1 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 10 trimming bilog shard 5 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.47.1 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 10 trimming bilog shard 6 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.30.1 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 10 trimming bilog shard 7 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.31.1 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 10 trimming bilog shard 8 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.40.1 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 10 trimming bilog shard 9 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.59.1 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 10 trimming bilog shard 10 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.24.1 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 run: stack=0x55c227678db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c227262770:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22767c040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 run: stack=0x55c227677bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c22768a1c0:op=0x55c227680a40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c22768bc00:op=0x55c22768f2f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c2276911c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c2276925b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c2276945e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c227262db0:op=0x55c2272656e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c227727860:op=0x55c22772cb50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c227693770:op=0x55c22772d9f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c227688de0:op=0x55c227262ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c22772b020:op=0x55c227727f70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227728770:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c227689720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c22768a1c0:op=0x55c227680a40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.0 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 run: stack=0x55c22768a1c0 is io blocked 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c22768bc00:op=0x55c22768f2f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.1 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 run: stack=0x55c22768bc00 is io blocked 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c2276911c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.2 2026-03-20T11:46:11.191 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 run: stack=0x55c227685830 is io blocked 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c2276925b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.3 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 run: stack=0x55c227687d10 is io blocked 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c2276945e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.4 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 run: stack=0x55c22767e040 is io blocked 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c227262db0:op=0x55c2272656e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.5 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 run: stack=0x55c227262db0 is io blocked 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 cr:s=0x55c227727860:op=0x55c22772cb50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.176+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.6 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 run: stack=0x55c227727860 is io blocked 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c227693770:op=0x55c22772d9f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.7 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 run: stack=0x55c227693770 is io blocked 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c227688de0:op=0x55c227262ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.8 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 run: stack=0x55c227688de0 is io blocked 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22772b020:op=0x55c227727f70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.9 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 run: stack=0x55c22772b020 is io blocked 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227728770:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.10 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 run: stack=0x55c227688970 is io blocked 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c227689720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 0d2386a4e59dfd1105d408aba9d3e2cd31c3647173e559a330a2538f3bb86a93 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:0d2386a4e59dfd1105d408aba9d3e2cd31c3647173e559a330a2538f3bb86a93 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 803709425676b72b62eba25aa1dfd2eb9b80e70cf0fd3be58bfac298d2d4f854 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=803709425676b72b62eba25aa1dfd2eb9b80e70cf0fd3be58bfac298d2d4f854 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=4, curl_handle=0x55c227681f00 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 run: stack=0x55c22767e740 is io blocked 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f72be7fc640 20 link_request req_data=0x55c227682650 req_data->id=4, curl_handle=0x55c227681f00 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22768d640:op=0x55c22768cc50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22768d640:op=0x55c22768cc50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22768d640:op=0x55c22768cc50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22768d640:op=0x55c22768cc50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 15 stack 0x55c22768d640 end 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 run: stack=0x55c22768d640 is done 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22725f210:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c2276864c0:op=0x55c227271250:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22725ffc0:op=0x55c227684b50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768a4f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c2276864c0:op=0x55c227271250:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22725ffc0:op=0x55c227684b50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768a4f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c2276864c0:op=0x55c227271250:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22725ffc0:op=0x55c227684b50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768a4f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c2276864c0:op=0x55c227271250:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 15 stack 0x55c2276864c0 end 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 run: stack=0x55c2276864c0 is done 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22725ffc0:op=0x55c227684b50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 15 stack 0x55c22725ffc0 end 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 run: stack=0x55c22725ffc0 is done 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768a4f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 15 stack 0x55c22768aee0 end 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 run: stack=0x55c22768aee0 is done 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22725f210:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22725f210:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22725f210:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c227677670:op=0x55c22767d310:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c227677670:op=0x55c22767d310:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c227677670:op=0x55c22767d310:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c227677670:op=0x55c22767d310:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c227677670:op=0x55c227270640:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 15 stack 0x55c227677670 end 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 run: stack=0x55c227677670 is done 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22758e610:op=0x55c227679aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 collect(): s=0x55c22758e610 stack=0x55c227677670 is complete 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 10 trimming bilogs for bucket=:dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) markers=[00000000002.33.1,00000000002.61.1,00000000002.33.1,00000000002.31.1,00000000002.38.1,00000000002.35.1,00000000002.63.1,00000000002.38.1,00000000002.34.1,00000000002.52.1,00000000002.51.1], shards=11 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22758e610:op=0x55c22768a4f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 10 trimming bilog shard 0 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.33.1 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 10 trimming bilog shard 1 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.61.1 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 10 trimming bilog shard 2 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.33.1 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 10 trimming bilog shard 3 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.31.1 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 10 trimming bilog shard 4 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.38.1 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 10 trimming bilog shard 5 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.35.1 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 10 trimming bilog shard 6 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.63.1 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 10 trimming bilog shard 7 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.38.1 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 10 trimming bilog shard 8 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.34.1 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 10 trimming bilog shard 9 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.52.1 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 10 trimming bilog shard 10 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.51.1 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 run: stack=0x55c22758e610 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c227677670:op=0x55c2272718b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768e0c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22725ffc0:op=0x55c22768fd80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c2276864c0:op=0x55c227691a60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22768d640:op=0x55c227751060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c2277343a0:op=0x55c227750370:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22774c210:op=0x55c22774e800:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22774cad0:op=0x55c22774f5a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c227731260:op=0x55c227754ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c227692260:op=0x55c2277552e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c227755d60:op=0x55c227751b40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c227677670:op=0x55c2272718b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.0 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 run: stack=0x55c227677670 is io blocked 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768e0c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.1 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 run: stack=0x55c22768aee0 is io blocked 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22725ffc0:op=0x55c22768fd80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.2 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 run: stack=0x55c22725ffc0 is io blocked 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c2276864c0:op=0x55c227691a60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.3 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 run: stack=0x55c2276864c0 is io blocked 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22768d640:op=0x55c227751060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.4 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 run: stack=0x55c22768d640 is io blocked 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c2277343a0:op=0x55c227750370:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.5 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 run: stack=0x55c2277343a0 is io blocked 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22774c210:op=0x55c22774e800:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.6 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 run: stack=0x55c22774c210 is io blocked 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c22774cad0:op=0x55c22774f5a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.7 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 run: stack=0x55c22774cad0 is io blocked 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c227731260:op=0x55c227754ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.8 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 run: stack=0x55c227731260 is io blocked 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c227692260:op=0x55c2277552e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.9 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 run: stack=0x55c227692260 is io blocked 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 cr:s=0x55c227755d60:op=0x55c227751b40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.10 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.177+0000 7f7363925900 20 run: stack=0x55c227755d60 is io blocked 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.178+0000 7f7363925900 20 cr:s=0x55c227727860:op=0x55c22772cb50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.178+0000 7f7363925900 20 cr:s=0x55c227727860:op=0x55c22772cb50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.178+0000 7f7363925900 20 cr:s=0x55c227727860:op=0x55c22772cb50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.178+0000 7f7363925900 20 cr:s=0x55c227727860:op=0x55c22772cb50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.178+0000 7f7363925900 15 stack 0x55c227727860 end 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.178+0000 7f7363925900 20 run: stack=0x55c227727860 is done 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.178+0000 7f7363925900 20 cr:s=0x55c227678db0:op=0x55c227270c60:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.178+0000 7f7363925900 20 run: stack=0x55c227678db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.179+0000 7f7363925900 20 cr:s=0x55c227262db0:op=0x55c2272656e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.179+0000 7f7363925900 20 cr:s=0x55c227262db0:op=0x55c2272656e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.179+0000 7f7363925900 20 cr:s=0x55c227262db0:op=0x55c2272656e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.179+0000 7f7363925900 20 cr:s=0x55c227262db0:op=0x55c2272656e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.179+0000 7f7363925900 15 stack 0x55c227262db0 end 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.179+0000 7f7363925900 20 run: stack=0x55c227262db0 is done 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.179+0000 7f7363925900 20 cr:s=0x55c227678db0:op=0x55c227270c60:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.179+0000 7f7363925900 20 run: stack=0x55c227678db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.179+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c2276911c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.179+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c2276911c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.179+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c2276911c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.179+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c2276911c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.179+0000 7f7363925900 15 stack 0x55c227685830 end 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.179+0000 7f7363925900 20 run: stack=0x55c227685830 is done 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.179+0000 7f7363925900 20 cr:s=0x55c227678db0:op=0x55c227270c60:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.179+0000 7f7363925900 20 run: stack=0x55c227678db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c2276925b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c2276925b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c2276925b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c2276925b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 15 stack 0x55c227687d10 end 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 run: stack=0x55c227687d10 is done 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 cr:s=0x55c227678db0:op=0x55c227270c60:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 run: stack=0x55c227678db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c227683f70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c227683f70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c227683f70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c227683f70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 15 stack 0x55c227260bc0 end 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 run: stack=0x55c227260bc0 is done 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22725f210:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c2276945e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c2276945e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227728770:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 cr:s=0x55c22768c920:op=0x55c22768bf30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c2276945e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227728770:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 cr:s=0x55c22768c920:op=0x55c22768bf30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c2276945e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 15 stack 0x55c22767e040 end 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 run: stack=0x55c22767e040 is done 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227728770:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 cr:s=0x55c22768c920:op=0x55c22768bf30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 cr:s=0x55c227678db0:op=0x55c227270c60:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 run: stack=0x55c227678db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227728770:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 15 stack 0x55c227688970 end 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 run: stack=0x55c227688970 is done 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 cr:s=0x55c22768c920:op=0x55c22768bf30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 15 stack 0x55c22768c920 end 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 run: stack=0x55c22768c920 is done 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 cr:s=0x55c227678db0:op=0x55c227270c60:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 run: stack=0x55c227678db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22725f210:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c227683f70:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c227683f70:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 enqueued request req=0x55c2273594e0 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 req: 0x55c2273594e0 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is io blocked 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f73257fa640 20 dequeued request req=0x55c2273594e0 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.180+0000 7f73257fa640 20 RGWWQ: empty 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c2277343a0:op=0x55c227750370:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c2277343a0:op=0x55c227750370:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c22774cad0:op=0x55c22774f5a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c227692260:op=0x55c2277552e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c2277343a0:op=0x55c227750370:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c22774cad0:op=0x55c22774f5a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c227692260:op=0x55c2277552e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c2277343a0:op=0x55c227750370:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 15 stack 0x55c2277343a0 end 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 run: stack=0x55c2277343a0 is done 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c22774cad0:op=0x55c22774f5a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c227692260:op=0x55c2277552e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c22758e610:op=0x55c22768a4f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 run: stack=0x55c22758e610 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c22774cad0:op=0x55c22774f5a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 15 stack 0x55c22774cad0 end 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 run: stack=0x55c22774cad0 is done 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c227692260:op=0x55c2277552e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 15 stack 0x55c227692260 end 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 run: stack=0x55c227692260 is done 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c22758e610:op=0x55c22768a4f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c22758e610:op=0x55c22768a4f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 run: stack=0x55c22758e610 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c22725ffc0:op=0x55c22768fd80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c22725ffc0:op=0x55c22768fd80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c2276864c0:op=0x55c227691a60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c22725ffc0:op=0x55c22768fd80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c2276864c0:op=0x55c227691a60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c22725ffc0:op=0x55c22768fd80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 15 stack 0x55c22725ffc0 end 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 run: stack=0x55c22725ffc0 is done 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c2276864c0:op=0x55c227691a60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c22758e610:op=0x55c22768a4f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 run: stack=0x55c22758e610 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c2276864c0:op=0x55c227691a60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 15 stack 0x55c2276864c0 end 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 run: stack=0x55c2276864c0 is done 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c22758e610:op=0x55c22768a4f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 run: stack=0x55c22758e610 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c22768a1c0:op=0x55c227680a40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c22768a1c0:op=0x55c227680a40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c227688de0:op=0x55c227262ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c22768d640:op=0x55c227751060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c227731260:op=0x55c227754ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c22768a1c0:op=0x55c227680a40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c227688de0:op=0x55c227262ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c22768d640:op=0x55c227751060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c227731260:op=0x55c227754ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c22768a1c0:op=0x55c227680a40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 15 stack 0x55c22768a1c0 end 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 run: stack=0x55c22768a1c0 is done 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c227688de0:op=0x55c227262ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c22768d640:op=0x55c227751060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c227731260:op=0x55c227754ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c227678db0:op=0x55c227270c60:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 run: stack=0x55c227678db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c227688de0:op=0x55c227262ef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 15 stack 0x55c227688de0 end 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 run: stack=0x55c227688de0 is done 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c22768d640:op=0x55c227751060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 15 stack 0x55c22768d640 end 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 run: stack=0x55c22768d640 is done 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c227731260:op=0x55c227754ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 15 stack 0x55c227731260 end 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 run: stack=0x55c227731260 is done 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c227678db0:op=0x55c227270c60:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 run: stack=0x55c227678db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c22758e610:op=0x55c22768a4f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c22758e610:op=0x55c22768a4f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 run: stack=0x55c22758e610 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c22768bc00:op=0x55c22768f2f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c22768bc00:op=0x55c22768f2f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c22768bc00:op=0x55c22768f2f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c22768bc00:op=0x55c22768f2f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 15 stack 0x55c22768bc00 end 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 run: stack=0x55c22768bc00 is done 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c227678db0:op=0x55c227270c60:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 run: stack=0x55c227678db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c227693770:op=0x55c22772d9f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c227693770:op=0x55c22772d9f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c227693770:op=0x55c22772d9f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c227693770:op=0x55c22772d9f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 15 stack 0x55c227693770 end 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 run: stack=0x55c227693770 is done 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c227678db0:op=0x55c227270c60:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 run: stack=0x55c227678db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c227677670:op=0x55c2272718b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c227677670:op=0x55c2272718b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c227677670:op=0x55c2272718b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c227677670:op=0x55c2272718b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 15 stack 0x55c227677670 end 2026-03-20T11:46:11.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 run: stack=0x55c227677670 is done 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c22758e610:op=0x55c22768a4f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 run: stack=0x55c22758e610 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f73257fa640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 info.flags=0x11 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c227755d60:op=0x55c227751b40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c227755d60:op=0x55c227751b40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c227755d60:op=0x55c227751b40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 cr:s=0x55c227755d60:op=0x55c227751b40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 15 stack 0x55c227755d60 end 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.181+0000 7f7363925900 20 run: stack=0x55c227755d60 is done 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22758e610:op=0x55c22768a4f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 run: stack=0x55c22758e610 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f73257fa640 10 rgw async rados processor: moving b1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 to cache LRU end 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f73257fa640 10 rgw async rados processor: distributing notification oid=b1.rgw.control:notify.6 cni=[op: 0, obj: b1.rgw.meta:root:.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20, ofs0, ns] 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22772b020:op=0x55c227727f70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22772b020:op=0x55c227727f70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22772b020:op=0x55c227727f70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22772b020:op=0x55c227727f70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 15 stack 0x55c22772b020 end 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 run: stack=0x55c22772b020 is done 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c227678db0:op=0x55c227270c60:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c227678db0:op=0x55c22767ad70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 trim: trimmed bucket instance dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 15 stack 0x55c227678db0 end 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 run: stack=0x55c227678db0 is done 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22767ad70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 4 starting trim on bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774dad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774dad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 enqueued request req=0x55c22725f800 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 req: 0x55c22725f800 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 run: stack=0x55c227688970 is io blocked 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731b7fe640 20 dequeued request req=0x55c22725f800 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731b7fe640 20 RGWWQ: empty 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731b7fe640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 : miss 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768e0c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768e0c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768e0c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768e0c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 15 stack 0x55c22768aee0 end 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 run: stack=0x55c22768aee0 is done 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22758e610:op=0x55c22768a4f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 run: stack=0x55c22758e610 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731b7fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731b7fe640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 info.flags=0x17 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731b7fe640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 to cache LRU end 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731b7fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731b7fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731b7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7339ffb640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 257698037996 cookie 94292373401888 notifier 4698 bl.length()=567 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731b7fe640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-31 : miss 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7339ffb640 10 rgw watcher librados: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 info.flags=0x11 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7339ffb640 10 rgw watcher librados: moving b1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 to cache LRU end 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731b7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731b7fe640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-31 info.flags=0x0 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731b7fe640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-31 to cache LRU end 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731b7fe640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-31 : miss 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22774c210:op=0x55c22774e800:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22774c210:op=0x55c22774e800:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22774c210:op=0x55c22774e800:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22774c210:op=0x55c22774e800:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 15 stack 0x55c22774c210 end 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 run: stack=0x55c22774c210 is done 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22758e610:op=0x55c22768a4f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22758e610:op=0x55c227679aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 trim: trimmed bucket instance dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 15 stack 0x55c22758e610 end 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 run: stack=0x55c22758e610 is done 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227679aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 4 starting trim on bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768e0c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768e0c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 enqueued request req=0x55c22768aae0 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 req: 0x55c22768aae0 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 run: stack=0x55c22768aee0 is io blocked 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731affd640 20 dequeued request req=0x55c22768aae0 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731affd640 20 RGWWQ: empty 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731affd640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 : miss 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731affd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731b7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731b7fe640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-31 info.flags=0x0 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731b7fe640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-31 to cache LRU end 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding source pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding dest pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f731b7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774dad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774dad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774dad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774dad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22767ad70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768e9a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22767ad70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 run: stack=0x55c227688970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c2272725c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c2272725c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f73257fa640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-60[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20]): orig_sources= new_sources= 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f73257fa640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-60[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20]): potential sources added=[] removed=[] 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f73257fa640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-60[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20]): orig_dests= new_dests= 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f73257fa640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-60[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20]): potential dests added=[] removed=[] 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.198 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 805ca205c317a29c2ec4accddae9df9e6f6c69a0be3002e9aa4cfa82f54aa9c9 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:805ca205c317a29c2ec4accddae9df9e6f6c69a0be3002e9aa4cfa82f54aa9c9 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 5848417c3a7090bc87f7aa0b7398f2bb1db8d0fbf0f5c1c2a2c7ad8bc120a8f1 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5848417c3a7090bc87f7aa0b7398f2bb1db8d0fbf0f5c1c2a2c7ad8bc120a8f1 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=5, curl_handle=0x55c227273060 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 run: stack=0x55c22767e040 is io blocked 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c227683f70:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c227683f70:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c227683f70:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c227683f70:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f72be7fc640 20 link_request req_data=0x55c22767dd80 req_data->id=5, curl_handle=0x55c227273060 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22774dad0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22774dad0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 enqueued request req=0x55c2271fbf40 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.182+0000 7f7363925900 20 req: 0x55c2271fbf40 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.183+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is io blocked 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.183+0000 7f731a7fc640 20 dequeued request req=0x55c2271fbf40 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.183+0000 7f731a7fc640 20 RGWWQ: empty 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.183+0000 7f731affd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.183+0000 7f731affd640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 info.flags=0x17 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.183+0000 7f731affd640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 to cache LRU end 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.183+0000 7f731affd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.183+0000 7f731affd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.183+0000 7f731affd640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.183+0000 7f731affd640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-39 : miss 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.183+0000 7f731affd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f731affd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f731affd640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-39 info.flags=0x0 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f731affd640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-39 to cache LRU end 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f731affd640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-39 : miss 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f731affd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f731affd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f731affd640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-39 info.flags=0x0 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f731affd640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-39 to cache LRU end 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f731a7fc640 10 rgw async rados processor: removing b1.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 from cache 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f731a7fc640 10 rgw async rados processor: distributing notification oid=b1.rgw.control:notify.6 cni=[op: 1, obj: b1.rgw.meta:root:.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20, ofs0, ns] 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f731affd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding source pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f731affd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding dest pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f731affd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f731affd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f731affd640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768e0c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768e0c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768e0c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768e0c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227679aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768e0c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227679aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 20 run: stack=0x55c22768aee0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c2276945e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c2276945e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = bf096a5a9bd7a276bd761f8846f7ee182ca7d019bd72561807421e79bd8de652 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:bf096a5a9bd7a276bd761f8846f7ee182ca7d019bd72561807421e79bd8de652 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = d91fe18fd9acca65561c3199939d1e1263f257865b331c27c08a9b5368eea9ac 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=d91fe18fd9acca65561c3199939d1e1263f257865b331c27c08a9b5368eea9ac 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=6, curl_handle=0x55c227264b20 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.184+0000 7f7363925900 20 run: stack=0x55c227260bc0 is io blocked 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f72be7fc640 20 link_request req_data=0x55c227265420 req_data->id=6, curl_handle=0x55c227264b20 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f733a7fc640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 257698037997 cookie 94292373401888 notifier 4698 bl.length()=206 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22774dad0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22774dad0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22774dad0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22774dad0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7363925900 20 trim: trimmed bucket instance dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7363925900 15 stack 0x55c22753d3c0 end 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is done 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7363925900 4 starting trim on bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7363925900 20 enqueued request req=0x55c227686910 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7363925900 20 req: 0x55c227686910 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is io blocked 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7319ffb640 20 dequeued request req=0x55c227686910 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7319ffb640 20 RGWWQ: empty 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7319ffb640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 : miss 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7319ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7319ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7319ffb640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 info.flags=0x17 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7319ffb640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 to cache LRU end 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7319ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7319ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7319ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7319ffb640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-8 : miss 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.185+0000 7f7319ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7319ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7319ffb640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-8 info.flags=0x0 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7319ffb640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-8 to cache LRU end 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7319ffb640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-8 : miss 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7319ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7319ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7319ffb640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-8 info.flags=0x0 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7319ffb640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-8 to cache LRU end 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7319ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding source pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7319ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding dest pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7319ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7319ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7319ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22774dad0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768f2f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768f2f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = fc1f3885895f81694b490e8ad7fe2ba70b48689816016d7b6bb9efe7bbda91c2 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:fc1f3885895f81694b490e8ad7fe2ba70b48689816016d7b6bb9efe7bbda91c2 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 32e50534a49ea9d272c0519912e63382f0ff995a682081c02f2b6dc407f08f02 2026-03-20T11:46:11.202 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=32e50534a49ea9d272c0519912e63382f0ff995a682081c02f2b6dc407f08f02 2026-03-20T11:46:11.202 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.202 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.202 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.202 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=7, curl_handle=0x55c2272606a0 2026-03-20T11:46:11.202 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f7363925900 20 run: stack=0x55c227687d10 is io blocked 2026-03-20T11:46:11.202 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.186+0000 7f72be7fc640 20 link_request req_data=0x7f72f8006330 req_data->id=7, curl_handle=0x55c2272606a0 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c227689720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c227689720:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c227262770:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f7363925900 15 stack 0x55c22767e740 end 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f7363925900 20 run: stack=0x55c22767e740 is done 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22767c040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f7363925900 20 collect(): s=0x55c227677bf0 stack=0x55c22767e740 is complete 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22767c040:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f7363925900 15 stack 0x55c227677bf0 end 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f7363925900 20 run: stack=0x55c227677bf0 is done 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f7363925900 4 starting trim on bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f7363925900 20 enqueued request req=0x55c22767f320 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f7363925900 20 req: 0x55c22767f320 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f7363925900 20 run: stack=0x55c227677bf0 is io blocked 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f73197fa640 20 dequeued request req=0x55c22767f320 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f73197fa640 20 RGWWQ: empty 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f73197fa640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 : miss 2026-03-20T11:46:11.220 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f73197fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f73197fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f73197fa640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 info.flags=0x17 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f73197fa640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 to cache LRU end 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f73197fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f73197fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f73197fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f73197fa640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-16 : miss 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f73197fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f73197fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f73197fa640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-16 info.flags=0x0 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f73197fa640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-16 to cache LRU end 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f73197fa640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-16 : miss 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.222+0000 7f73197fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f73197fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f73197fa640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-16 info.flags=0x0 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f73197fa640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-16 to cache LRU end 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f73197fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding source pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f73197fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding dest pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f73197fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f73197fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f73197fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c227262770:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 20 run: stack=0x55c227677bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768a1a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768a1a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 15a9402dd2203de104484b71308d6c07d1a2af760fb559d9ae65b1622b4748b3 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:15a9402dd2203de104484b71308d6c07d1a2af760fb559d9ae65b1622b4748b3 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 2a9ff4c09448d4d8c429c1ef4b4fb8096ca762122502114b5b570ad4ef9098b3 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2a9ff4c09448d4d8c429c1ef4b4fb8096ca762122502114b5b570ad4ef9098b3 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=8, curl_handle=0x55c227681f00 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f7363925900 20 run: stack=0x55c22767e740 is io blocked 2026-03-20T11:46:11.222 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.223+0000 7f72be7fc640 20 link_request req_data=0x55c227682650 req_data->id=8, curl_handle=0x55c227681f00 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c2272725c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c2272725c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768e9a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7363925900 15 stack 0x55c22767e040 end 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7363925900 20 run: stack=0x55c22767e040 is done 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22767ad70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7363925900 20 collect(): s=0x55c227688970 stack=0x55c22767e040 is complete 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22767ad70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7363925900 15 stack 0x55c227688970 end 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7363925900 20 run: stack=0x55c227688970 is done 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22772e7d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7363925900 4 starting trim on bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7363925900 20 enqueued request req=0x55c22725f800 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7363925900 20 req: 0x55c22725f800 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7363925900 20 run: stack=0x55c227688970 is io blocked 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7318ff9640 20 dequeued request req=0x55c22725f800 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7318ff9640 20 RGWWQ: empty 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7318ff9640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 : miss 2026-03-20T11:46:11.223 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7318ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.224 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.225+0000 7f3e3c85f640 -1 req 2655508908273431528 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7318ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7318ff9640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 info.flags=0x17 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7318ff9640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 to cache LRU end 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7318ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7318ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7318ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7318ff9640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-43 : miss 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7318ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7318ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7318ff9640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-43 info.flags=0x0 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7318ff9640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-43 to cache LRU end 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7318ff9640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-43 : miss 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.225+0000 7f7318ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7318ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7318ff9640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-43 info.flags=0x0 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7318ff9640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-43 to cache LRU end 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7318ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding source pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7318ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding dest pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7318ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7318ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7318ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22772e7d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768e9a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22772e7d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 20 run: stack=0x55c227688970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c2272725c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c2272725c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.224 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = baa49b3d14dbf9b5b450647f00ceea6f3922bdaa324a60feea6853f09027f4a4 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:baa49b3d14dbf9b5b450647f00ceea6f3922bdaa324a60feea6853f09027f4a4 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 97c405de43e7c1440918d8882ef74dc1a797c7748e5346d5285e4070ed05c5be 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=97c405de43e7c1440918d8882ef74dc1a797c7748e5346d5285e4070ed05c5be 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=9, curl_handle=0x55c227273060 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f7363925900 20 run: stack=0x55c22767e040 is io blocked 2026-03-20T11:46:11.225 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.226+0000 7f72be7fc640 20 link_request req_data=0x55c22767dd80 req_data->id=9, curl_handle=0x55c227273060 2026-03-20T11:46:11.225 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.227+0000 7f3ea9939640 -1 req 139675104522044400 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c2276945e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c2276945e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768e0c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f7363925900 15 stack 0x55c227260bc0 end 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f7363925900 20 run: stack=0x55c227260bc0 is done 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227679aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f7363925900 20 collect(): s=0x55c22768aee0 stack=0x55c227260bc0 is complete 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227679aa0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f7363925900 15 stack 0x55c22768aee0 end 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f7363925900 20 run: stack=0x55c22768aee0 is done 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f7363925900 4 starting trim on bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768e0c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768e0c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f7363925900 20 enqueued request req=0x55c22768aae0 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f7363925900 20 req: 0x55c22768aae0 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f7363925900 20 run: stack=0x55c22768aee0 is io blocked 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f73187f8640 20 dequeued request req=0x55c22768aae0 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f73187f8640 20 RGWWQ: empty 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f73187f8640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 : miss 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f73187f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f73187f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f73187f8640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 info.flags=0x17 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f73187f8640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 to cache LRU end 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f73187f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f73187f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f73187f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f73187f8640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-10 : miss 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f73187f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f73187f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f73187f8640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-10 info.flags=0x0 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f73187f8640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-10 to cache LRU end 2026-03-20T11:46:11.226 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f73187f8640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-10 : miss 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f73187f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f73187f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f73187f8640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-10 info.flags=0x0 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f73187f8640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-10 to cache LRU end 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f73187f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding source pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f73187f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding dest pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f73187f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f73187f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f73187f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.227+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768e0c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768e0c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768e0c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768e0c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768e0c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f7363925900 20 run: stack=0x55c22768aee0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c2276945e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c2276945e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 9910ee689755599d3c35adcbaefb98b9c4bd3acffb3fbf3706b26bf29e6135bf 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:9910ee689755599d3c35adcbaefb98b9c4bd3acffb3fbf3706b26bf29e6135bf 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = e67e2643e560cd72698e1c550d51743baa500bfdad4bfb8ce3a1cea63e5c533d 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=e67e2643e560cd72698e1c550d51743baa500bfdad4bfb8ce3a1cea63e5c533d 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.228 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.228 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=10, curl_handle=0x55c227264b20 2026-03-20T11:46:11.228 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f7363925900 20 run: stack=0x55c227260bc0 is io blocked 2026-03-20T11:46:11.228 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.228+0000 7f72be7fc640 20 link_request req_data=0x55c227265420 req_data->id=10, curl_handle=0x55c227264b20 2026-03-20T11:46:11.228 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.229+0000 7f3ec296b640 -1 req 6806651703404357372 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.228 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768f2f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.228 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768f2f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.228 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22774dad0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.228 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 15 stack 0x55c227687d10 end 2026-03-20T11:46:11.228 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 run: stack=0x55c227687d10 is done 2026-03-20T11:46:11.228 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.228 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 collect(): s=0x55c22753d3c0 stack=0x55c227687d10 is complete 2026-03-20T11:46:11.228 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.228 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.228 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.228 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.228 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 15 stack 0x55c22753d3c0 end 2026-03-20T11:46:11.228 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.228 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is done 2026-03-20T11:46:11.228 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.228 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.228 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.228 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.228 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 4 starting trim on bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 enqueued request req=0x55c227686910 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 req: 0x55c227686910 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is io blocked 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7317ff7640 20 dequeued request req=0x55c227686910 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7317ff7640 20 RGWWQ: empty 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7317ff7640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 : miss 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7317ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7317ff7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7317ff7640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 info.flags=0x17 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7317ff7640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 to cache LRU end 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7317ff7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7317ff7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7317ff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7317ff7640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-3 : miss 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7317ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7317ff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7317ff7640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-3 info.flags=0x0 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7317ff7640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-3 to cache LRU end 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7317ff7640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-3 : miss 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7317ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7317ff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7317ff7640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-3 info.flags=0x0 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7317ff7640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-3 to cache LRU end 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7317ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding source pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7317ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding dest pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7317ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7317ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7317ff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22774dad0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768f2f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768f2f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = d7bf5e2f871f223a189b5a875658281f72465998851cb8a25d6e06f78c99ad6e 2026-03-20T11:46:11.229 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.230 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.230 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.230 INFO:teuthology.orchestra.run.vm07.stderr:d7bf5e2f871f223a189b5a875658281f72465998851cb8a25d6e06f78c99ad6e 2026-03-20T11:46:11.230 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.230 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.230 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.230 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.230 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = c2c0a0978bffaf85b4b7c3160c9330db83391e2d6d49ef29bf4899590c763db7 2026-03-20T11:46:11.230 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c2c0a0978bffaf85b4b7c3160c9330db83391e2d6d49ef29bf4899590c763db7 2026-03-20T11:46:11.230 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.230 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.230 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.230 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=11, curl_handle=0x55c2272606a0 2026-03-20T11:46:11.230 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f7363925900 20 run: stack=0x55c227687d10 is io blocked 2026-03-20T11:46:11.230 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.229+0000 7f72be7fc640 20 link_request req_data=0x7f72f8006330 req_data->id=11, curl_handle=0x55c2272606a0 2026-03-20T11:46:11.230 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.231+0000 7f3ec1969640 -1 req 4380761716966459777 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768a1a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768a1a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c227262770:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7363925900 15 stack 0x55c22767e740 end 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7363925900 20 run: stack=0x55c22767e740 is done 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7363925900 20 collect(): s=0x55c227677bf0 stack=0x55c22767e740 is complete 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7363925900 15 stack 0x55c227677bf0 end 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7363925900 20 run: stack=0x55c227677bf0 is done 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7363925900 4 starting trim on bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7363925900 20 enqueued request req=0x55c22767f320 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7363925900 20 req: 0x55c22767f320 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7363925900 20 run: stack=0x55c227677bf0 is io blocked 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7316ff5640 20 dequeued request req=0x55c22767f320 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7316ff5640 20 RGWWQ: empty 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7316ff5640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 : miss 2026-03-20T11:46:11.264 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7316ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7316ff5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7316ff5640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 info.flags=0x17 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7316ff5640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 to cache LRU end 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7316ff5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7316ff5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7316ff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7316ff5640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-9 : miss 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7316ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7316ff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7316ff5640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-9 info.flags=0x0 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.266+0000 7f7316ff5640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-9 to cache LRU end 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7316ff5640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-9 : miss 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7316ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7316ff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7316ff5640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-9 info.flags=0x0 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7316ff5640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-9 to cache LRU end 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7316ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding source pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7316ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding dest pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7316ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7316ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7316ff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c227262770:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 20 run: stack=0x55c227677bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c2276907c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c2276907c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.266 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.266 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.266 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.266 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.266 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.266 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.266 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.266 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.266 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.266 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.266 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = ef5a2332c69b62b82026de54360f974147536b9c5eaccf786306c4dc9df9e211 2026-03-20T11:46:11.266 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.266 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.266 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.266 INFO:teuthology.orchestra.run.vm07.stderr:ef5a2332c69b62b82026de54360f974147536b9c5eaccf786306c4dc9df9e211 2026-03-20T11:46:11.266 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.266 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.266 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.266 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.266 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 14b6be1478aa45ac6d4799ffd07d4f707d8389628cfe3cee6c6e1fef8bbc50c4 2026-03-20T11:46:11.266 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=14b6be1478aa45ac6d4799ffd07d4f707d8389628cfe3cee6c6e1fef8bbc50c4 2026-03-20T11:46:11.266 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.266 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.266 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.266 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=12, curl_handle=0x55c227681f00 2026-03-20T11:46:11.266 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f7363925900 20 run: stack=0x55c22767e740 is io blocked 2026-03-20T11:46:11.266 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.267+0000 7f72be7fc640 20 link_request req_data=0x55c227682650 req_data->id=12, curl_handle=0x55c227681f00 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c2272725c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c2272725c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768e9a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f7363925900 15 stack 0x55c22767e040 end 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f7363925900 20 run: stack=0x55c22767e040 is done 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22772e7d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f7363925900 20 collect(): s=0x55c227688970 stack=0x55c22767e040 is complete 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22772e7d0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f7363925900 15 stack 0x55c227688970 end 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f7363925900 20 run: stack=0x55c227688970 is done 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f7363925900 4 starting trim on bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f7363925900 20 enqueued request req=0x55c22725f800 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f7363925900 20 req: 0x55c22725f800 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f7363925900 20 run: stack=0x55c227688970 is io blocked 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f73167f4640 20 dequeued request req=0x55c22725f800 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f73167f4640 20 RGWWQ: empty 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f73167f4640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 : miss 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f73167f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f73167f4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f73167f4640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 info.flags=0x17 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f73167f4640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 to cache LRU end 2026-03-20T11:46:11.267 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f73167f4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f73167f4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f73167f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f73167f4640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-38 : miss 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f73167f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f73167f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f73167f4640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-38 info.flags=0x0 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f73167f4640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-38 to cache LRU end 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f73167f4640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-38 : miss 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f73167f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f73167f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f73167f4640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-38 info.flags=0x0 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f73167f4640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-38 to cache LRU end 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f73167f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding source pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f73167f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding dest pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f73167f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f73167f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.268+0000 7f73167f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768e9a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 20 run: stack=0x55c227688970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c2272725c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c2272725c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 2459189ae12840ef4fa35280e81e56c17fe83e7e6104c0cde15c399e55d03116 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2459189ae12840ef4fa35280e81e56c17fe83e7e6104c0cde15c399e55d03116 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 680030059a0b9ddb0eda404b6964a59288f7baab8c54f220ad095ce4498f3497 2026-03-20T11:46:11.268 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=680030059a0b9ddb0eda404b6964a59288f7baab8c54f220ad095ce4498f3497 2026-03-20T11:46:11.269 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.269 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.269 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.269 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=13, curl_handle=0x55c227273060 2026-03-20T11:46:11.269 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f7363925900 20 run: stack=0x55c22767e040 is io blocked 2026-03-20T11:46:11.269 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.269+0000 7f72be7fc640 20 link_request req_data=0x55c22767dd80 req_data->id=13, curl_handle=0x55c227273060 2026-03-20T11:46:11.269 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.269+0000 7f3e06ff4640 -1 req 17784872816077216903 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.269 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.270+0000 7f3ead941640 -1 req 3385827531601206497 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.269 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c2276945e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.269 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c2276945e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.269 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768e0c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.269 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7363925900 15 stack 0x55c227260bc0 end 2026-03-20T11:46:11.269 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7363925900 20 run: stack=0x55c227260bc0 is done 2026-03-20T11:46:11.269 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.269 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7363925900 20 collect(): s=0x55c22768aee0 stack=0x55c227260bc0 is complete 2026-03-20T11:46:11.269 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.269 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.269 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.269 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.269 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7363925900 15 stack 0x55c22768aee0 end 2026-03-20T11:46:11.269 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7363925900 20 run: stack=0x55c22768aee0 is done 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7363925900 4 starting trim on bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768e0c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768e0c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7363925900 20 enqueued request req=0x55c22768aae0 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7363925900 20 req: 0x55c22768aae0 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7363925900 20 run: stack=0x55c22768aee0 is io blocked 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7315ff3640 20 dequeued request req=0x55c22768aae0 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7315ff3640 20 RGWWQ: empty 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7315ff3640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 : miss 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7315ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7315ff3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7315ff3640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 info.flags=0x17 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7315ff3640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 to cache LRU end 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7315ff3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7315ff3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7315ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7315ff3640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-15 : miss 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7315ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7315ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7315ff3640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-15 info.flags=0x0 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7315ff3640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-15 to cache LRU end 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7315ff3640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-15 : miss 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.270+0000 7f7315ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7315ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7315ff3640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-15 info.flags=0x0 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7315ff3640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-15 to cache LRU end 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7315ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding source pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7315ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding dest pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7315ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7315ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7315ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768e0c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768e0c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768e0c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768e0c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c2276945e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 20 run: stack=0x55c22768aee0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768e0c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768e0c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.270 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.271 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.271 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.271 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.271 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.271 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.271 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.271 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.271 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.271 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 5009408474b9c47aafac3e21b8e383786aa689792d06ab23878252c7ea5bb229 2026-03-20T11:46:11.271 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.271 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.271 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.271 INFO:teuthology.orchestra.run.vm07.stderr:5009408474b9c47aafac3e21b8e383786aa689792d06ab23878252c7ea5bb229 2026-03-20T11:46:11.271 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.271 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.271 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.271 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.271 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 456df857e69591be329ce3fce08cbd001d22bb5ab24d0c6f44d67a6c08cc088d 2026-03-20T11:46:11.271 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=456df857e69591be329ce3fce08cbd001d22bb5ab24d0c6f44d67a6c08cc088d 2026-03-20T11:46:11.271 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.271 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.271 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.271 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=14, curl_handle=0x55c227264b20 2026-03-20T11:46:11.271 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f7363925900 20 run: stack=0x55c227260bc0 is io blocked 2026-03-20T11:46:11.271 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.271+0000 7f72be7fc640 20 link_request req_data=0x55c227265420 req_data->id=14, curl_handle=0x55c227264b20 2026-03-20T11:46:11.271 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.273+0000 7f3dfbfde640 -1 req 15875797028784290965 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.271 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768f2f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.271 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768f2f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22774dad0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 15 stack 0x55c227687d10 end 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 20 run: stack=0x55c227687d10 is done 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 20 collect(): s=0x55c22753d3c0 stack=0x55c227687d10 is complete 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 15 stack 0x55c22753d3c0 end 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is done 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 4 starting trim on bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768fd80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768fd80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 20 enqueued request req=0x55c227686910 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 20 req: 0x55c227686910 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is io blocked 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f73157f2640 20 dequeued request req=0x55c227686910 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f73157f2640 20 RGWWQ: empty 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f73157f2640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 : miss 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f73157f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f73157f2640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f73157f2640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 info.flags=0x17 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f73157f2640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 to cache LRU end 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f73157f2640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f73157f2640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f73157f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f73157f2640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-13 : miss 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f73157f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f73157f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f73157f2640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-13 info.flags=0x0 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f73157f2640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-13 to cache LRU end 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f73157f2640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-13 : miss 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f73157f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f73157f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f73157f2640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-13 info.flags=0x0 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f73157f2640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-13 to cache LRU end 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f73157f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18): adding source pipe: {s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f73157f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18): adding dest pipe: {s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f73157f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f73157f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f73157f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768fd80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768fd80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768fd80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768fd80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.272+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.273+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22774dad0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.273+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.273+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.273+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768fd80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.273+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768fd80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.272 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.273+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.273+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.273+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.273+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.273+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.273+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.273+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = cfe3b24a27fa759549dbadddb0d8dfae26ba95f888fc154f63becfa197d1312d 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.273+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:cfe3b24a27fa759549dbadddb0d8dfae26ba95f888fc154f63becfa197d1312d 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.273+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.273+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.273+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.273+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.273+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 9ccf5898cc09fc20ebaf688e5ad37069f6bfaab86d0eb1c7a2f00fba72f924ac 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.273+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=9ccf5898cc09fc20ebaf688e5ad37069f6bfaab86d0eb1c7a2f00fba72f924ac 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.273+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.273+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.273+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.273+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=15, curl_handle=0x55c2272606a0 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.273+0000 7f7363925900 20 run: stack=0x55c227687d10 is io blocked 2026-03-20T11:46:11.273 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.273+0000 7f72be7fc640 20 link_request req_data=0x7f72f8006330 req_data->id=15, curl_handle=0x55c2272606a0 2026-03-20T11:46:11.273 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.274+0000 7f3e840ee640 -1 req 6777849283444426391 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.308 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c2276907c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.308 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c2276907c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.308 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c227262770:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.308 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7363925900 15 stack 0x55c22767e740 end 2026-03-20T11:46:11.308 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7363925900 20 run: stack=0x55c22767e740 is done 2026-03-20T11:46:11.308 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.308 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7363925900 20 collect(): s=0x55c227677bf0 stack=0x55c22767e740 is complete 2026-03-20T11:46:11.308 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.308 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.308 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.308 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.308 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7363925900 15 stack 0x55c227677bf0 end 2026-03-20T11:46:11.308 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.308 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7363925900 20 run: stack=0x55c227677bf0 is done 2026-03-20T11:46:11.308 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.308 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.308 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.308 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.308 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7363925900 4 starting trim on bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:11.308 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.308 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.308 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7363925900 20 enqueued request req=0x55c22767f320 2026-03-20T11:46:11.308 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.308 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7363925900 20 req: 0x55c22767f320 2026-03-20T11:46:11.308 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7363925900 20 run: stack=0x55c227677bf0 is io blocked 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7314ff1640 20 dequeued request req=0x55c22767f320 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7314ff1640 20 RGWWQ: empty 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7314ff1640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 : miss 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7314ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7314ff1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7314ff1640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 info.flags=0x17 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7314ff1640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 to cache LRU end 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7314ff1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7314ff1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7314ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7314ff1640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-11 : miss 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7314ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7314ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7314ff1640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-11 info.flags=0x0 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7314ff1640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-11 to cache LRU end 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7314ff1640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-11 : miss 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.310+0000 7f7314ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7314ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7314ff1640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-11 info.flags=0x0 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c2272725c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7314ff1640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-11 to cache LRU end 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c2272725c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768e9a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 15 stack 0x55c22767e040 end 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 run: stack=0x55c22767e040 is done 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7314ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16): adding source pipe: {s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 collect(): s=0x55c227688970 stack=0x55c22767e040 is complete 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 15 stack 0x55c227688970 end 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 run: stack=0x55c227688970 is done 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7314ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16): adding dest pipe: {s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.309 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 4 starting trim on bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 enqueued request req=0x55c22725f800 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 req: 0x55c22725f800 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 run: stack=0x55c227688970 is io blocked 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7314ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7314ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7314ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f73147f0640 20 dequeued request req=0x55c22725f800 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f73147f0640 20 RGWWQ: empty 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f73147f0640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 : miss 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f73147f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768a1a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 run: stack=0x55c227677bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 3a124960fc6ec8408ae9dd7fb4167019923b520fabb41075a6a58bf027021f57 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:3a124960fc6ec8408ae9dd7fb4167019923b520fabb41075a6a58bf027021f57 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 33ea5ba4d307771f33cbe1a1e2c915532bf2beb8576861fb9d5f044c135a5960 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=33ea5ba4d307771f33cbe1a1e2c915532bf2beb8576861fb9d5f044c135a5960 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=16, curl_handle=0x55c227273060 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f7363925900 20 run: stack=0x55c22767e040 is io blocked 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f72be7fc640 20 link_request req_data=0x55c22767dd80 req_data->id=16, curl_handle=0x55c227273060 2026-03-20T11:46:11.310 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f73147f0640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f73147f0640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 info.flags=0x17 2026-03-20T11:46:11.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f73147f0640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 to cache LRU end 2026-03-20T11:46:11.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f73147f0640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f73147f0640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f73147f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:11.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f73147f0640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-12 : miss 2026-03-20T11:46:11.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f73147f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.311 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.313+0000 7f3e90106640 -1 req 6206362076423243635 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f73147f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f73147f0640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-12 info.flags=0x0 2026-03-20T11:46:11.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f73147f0640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-12 to cache LRU end 2026-03-20T11:46:11.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f73147f0640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-12 : miss 2026-03-20T11:46:11.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.311+0000 7f73147f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f73147f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f73147f0640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-12 info.flags=0x0 2026-03-20T11:46:11.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f73147f0640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-12 to cache LRU end 2026-03-20T11:46:11.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f73147f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17): adding source pipe: {s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f73147f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17): adding dest pipe: {s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f73147f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f73147f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f73147f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:11.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c2272718b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 20 run: stack=0x55c227688970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c227727f70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c227727f70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = b523e46a8d30aee2eb8d6139553191bc83aac51a6eb5c35a7c495adfb426b60b 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:b523e46a8d30aee2eb8d6139553191bc83aac51a6eb5c35a7c495adfb426b60b 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 3f10640c93311a0384288ebadc592c2a65ad09c334a2ded9f22534eec31b7192 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=3f10640c93311a0384288ebadc592c2a65ad09c334a2ded9f22534eec31b7192 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=17, curl_handle=0x55c227681f00 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f7363925900 20 run: stack=0x55c22767e740 is io blocked 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.312+0000 7f72be7fc640 20 link_request req_data=0x55c227682650 req_data->id=17, curl_handle=0x55c227681f00 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768e0c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768e0c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c2276945e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7363925900 15 stack 0x55c227260bc0 end 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7363925900 20 run: stack=0x55c227260bc0 is done 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7363925900 20 collect(): s=0x55c22768aee0 stack=0x55c227260bc0 is complete 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7363925900 15 stack 0x55c22768aee0 end 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7363925900 20 run: stack=0x55c22768aee0 is done 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7363925900 4 starting trim on bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7363925900 20 enqueued request req=0x55c22768aae0 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7363925900 20 req: 0x55c22768aae0 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7363925900 20 run: stack=0x55c22768aee0 is io blocked 2026-03-20T11:46:11.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7313fef640 20 dequeued request req=0x55c22768aae0 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7313fef640 20 RGWWQ: empty 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7313fef640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 : miss 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7313fef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7313fef640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7313fef640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 info.flags=0x17 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7313fef640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 to cache LRU end 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7313fef640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7313fef640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7313fef640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7313fef640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-40 : miss 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7313fef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.313 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.314+0000 7f3e9290b640 -1 req 18128827275545547046 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7313fef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7313fef640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-40 info.flags=0x0 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7313fef640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-40 to cache LRU end 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7313fef640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-40 : miss 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.313+0000 7f7313fef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7313fef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7313fef640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-40 info.flags=0x0 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7313fef640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-40 to cache LRU end 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7313fef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1): adding source pipe: {s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7313fef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1): adding dest pipe: {s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7313fef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7313fef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7313fef640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c2276945e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 20 run: stack=0x55c22768aee0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768e0c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768e0c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 19d23494c87d1ac54e8da37ad4eb53ea4e421209434785ab82778dc3500a92aa 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:19d23494c87d1ac54e8da37ad4eb53ea4e421209434785ab82778dc3500a92aa 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = a2e01e7e7259e0c863e487c4c63d83711cef38903af25918787de6e8d3ca2ca2 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a2e01e7e7259e0c863e487c4c63d83711cef38903af25918787de6e8d3ca2ca2 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=18, curl_handle=0x55c227264b20 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f7363925900 20 run: stack=0x55c227260bc0 is io blocked 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.314+0000 7f72be7fc640 20 link_request req_data=0x55c227265420 req_data->id=18, curl_handle=0x55c227264b20 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768fd80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768fd80:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22774dad0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f7363925900 15 stack 0x55c227687d10 end 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f7363925900 20 run: stack=0x55c227687d10 is done 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f7363925900 20 collect(): s=0x55c22753d3c0 stack=0x55c227687d10 is complete 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f7363925900 15 stack 0x55c22753d3c0 end 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is done 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f7363925900 4 starting trim on bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f7363925900 20 enqueued request req=0x55c227686910 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f7363925900 20 req: 0x55c227686910 2026-03-20T11:46:11.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is io blocked 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f73137ee640 20 dequeued request req=0x55c227686910 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f73137ee640 20 RGWWQ: empty 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f73137ee640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 : miss 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f73137ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f73137ee640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f73137ee640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 info.flags=0x17 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f73137ee640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 to cache LRU end 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f73137ee640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f73137ee640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f73137ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f73137ee640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-18 : miss 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f73137ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.315 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.316+0000 7f3e9e122640 -1 req 1794868448994722888 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f73137ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f73137ee640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-18 info.flags=0x0 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f73137ee640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-18 to cache LRU end 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f73137ee640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-18 : miss 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.315+0000 7f73137ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f73137ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f73137ee640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-18 info.flags=0x0 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f73137ee640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-18 to cache LRU end 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f73137ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23): adding source pipe: {s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f73137ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23): adding dest pipe: {s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f73137ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f73137ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f73137ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768fd80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22774da60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22774da60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.315 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.316 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.316 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.316 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.316 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.316 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.316 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.316 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.316 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.316 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.316 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.316 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.316 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 88bcf0f075e5ae05b8b0f4b41906e2767afd1801d7822842f61b3a8162b42f4e 2026-03-20T11:46:11.316 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.316 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.316 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.316 INFO:teuthology.orchestra.run.vm07.stderr:88bcf0f075e5ae05b8b0f4b41906e2767afd1801d7822842f61b3a8162b42f4e 2026-03-20T11:46:11.316 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.316 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.316 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.316 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.316 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = b793d6ddc2e6b7cd7879cf86a69d0606144bd3ad1e563772d26aaad9fe9e9976 2026-03-20T11:46:11.316 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b793d6ddc2e6b7cd7879cf86a69d0606144bd3ad1e563772d26aaad9fe9e9976 2026-03-20T11:46:11.316 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.316 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.316 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.316 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=19, curl_handle=0x55c2272606a0 2026-03-20T11:46:11.316 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f7363925900 20 run: stack=0x55c227687d10 is io blocked 2026-03-20T11:46:11.316 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.316+0000 7f72be7fc640 20 link_request req_data=0x7f72f8006330 req_data->id=19, curl_handle=0x55c2272606a0 2026-03-20T11:46:11.316 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.317+0000 7f3e6a8bb640 -1 req 16640230945243029169 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768a1a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7363925900 15 stack 0x55c22767e040 end 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7363925900 20 run: stack=0x55c22767e040 is done 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7363925900 20 collect(): s=0x55c227677bf0 stack=0x55c22767e040 is complete 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7363925900 15 stack 0x55c227677bf0 end 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7363925900 20 run: stack=0x55c227677bf0 is done 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7363925900 4 starting trim on bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7363925900 20 enqueued request req=0x55c22767f320 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7363925900 20 req: 0x55c22767f320 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7363925900 20 run: stack=0x55c227677bf0 is io blocked 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7312fed640 20 dequeued request req=0x55c22767f320 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7312fed640 20 RGWWQ: empty 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7312fed640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 : miss 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7312fed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7312fed640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.352 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7312fed640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 info.flags=0x17 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7312fed640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 to cache LRU end 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7312fed640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7312fed640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7312fed640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7312fed640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-50 : miss 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.354+0000 7f7312fed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7312fed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7312fed640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-50 info.flags=0x0 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7312fed640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-50 to cache LRU end 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7312fed640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-50 : miss 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7312fed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c227727f70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c227727f70:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c2272718b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 15 stack 0x55c22767e740 end 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 run: stack=0x55c22767e740 is done 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 collect(): s=0x55c227688970 stack=0x55c22767e740 is complete 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 15 stack 0x55c227688970 end 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 run: stack=0x55c227688970 is done 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 4 starting trim on bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227727f70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227727f70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 enqueued request req=0x55c22725f800 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 req: 0x55c22725f800 2026-03-20T11:46:11.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 run: stack=0x55c227688970 is io blocked 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7312fed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7312fed640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-50 info.flags=0x0 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7312fed640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-50 to cache LRU end 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7312fed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10): adding source pipe: {s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7312fed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10): adding dest pipe: {s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7312fed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f73127ec640 20 dequeued request req=0x55c22725f800 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7312fed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f73127ec640 20 RGWWQ: empty 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7312fed640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f73127ec640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 : miss 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f73127ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c2272718b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 run: stack=0x55c227677bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 400a850c94c961db6d750ae76ca6a0ef6305dc9503a20e2eb164744ea5729e79 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:400a850c94c961db6d750ae76ca6a0ef6305dc9503a20e2eb164744ea5729e79 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = d1f7218140a79f55541cd059e4e2031055bdaffbbae2358f249b1ff433f7eb9f 2026-03-20T11:46:11.355 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=d1f7218140a79f55541cd059e4e2031055bdaffbbae2358f249b1ff433f7eb9f 2026-03-20T11:46:11.355 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.355 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.355 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.355 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=20, curl_handle=0x55c227681f00 2026-03-20T11:46:11.355 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f7363925900 20 run: stack=0x55c22767e740 is io blocked 2026-03-20T11:46:11.355 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f72be7fc640 20 link_request req_data=0x55c227682650 req_data->id=20, curl_handle=0x55c227681f00 2026-03-20T11:46:11.355 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f73127ec640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.355 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f73127ec640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 info.flags=0x17 2026-03-20T11:46:11.355 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f73127ec640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 to cache LRU end 2026-03-20T11:46:11.355 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f73127ec640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.355 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f73127ec640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.355 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f73127ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:11.355 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f73127ec640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-24 : miss 2026-03-20T11:46:11.355 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.355+0000 7f73127ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.355 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.357+0000 7f3e8e903640 -1 req 17661254327628210221 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.355 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768e0c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.355 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768e0c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.355 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c2276945e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.355 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 15 stack 0x55c227260bc0 end 2026-03-20T11:46:11.355 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 run: stack=0x55c227260bc0 is done 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 collect(): s=0x55c22768aee0 stack=0x55c227260bc0 is complete 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 15 stack 0x55c22768aee0 end 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 run: stack=0x55c22768aee0 is done 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 4 starting trim on bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2277286c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2277286c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 enqueued request req=0x55c22768aae0 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 req: 0x55c22768aae0 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 run: stack=0x55c22768aee0 is io blocked 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7311feb640 20 dequeued request req=0x55c22768aae0 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7311feb640 20 RGWWQ: empty 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7311feb640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 : miss 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7311feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f73127ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f73127ec640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-24 info.flags=0x0 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f73127ec640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-24 to cache LRU end 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f73127ec640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-24 : miss 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f73127ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7311feb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7311feb640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 info.flags=0x17 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7311feb640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 to cache LRU end 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7311feb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7311feb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7311feb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7311feb640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-56 : miss 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7311feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f73127ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f73127ec640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-24 info.flags=0x0 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f73127ec640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-24 to cache LRU end 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f73127ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29): adding source pipe: {s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f73127ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29): adding dest pipe: {s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f73127ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f73127ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f73127ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227727f70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227727f70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227727f70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227727f70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c2276945e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 run: stack=0x55c227688970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768e0c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768e0c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 176bc276f7404075f751baa32ff5b36d4c5115275c0a3f1cbc28abfb6c8db559 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:176bc276f7404075f751baa32ff5b36d4c5115275c0a3f1cbc28abfb6c8db559 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = eeece22d899da84a4d757507c7a40adf3ae8debca04d2174a560916bf6008b49 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=eeece22d899da84a4d757507c7a40adf3ae8debca04d2174a560916bf6008b49 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=21, curl_handle=0x55c227264b20 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7363925900 20 run: stack=0x55c227260bc0 is io blocked 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f72be7fc640 20 link_request req_data=0x55c227265420 req_data->id=21, curl_handle=0x55c227264b20 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7311feb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7311feb640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-56 info.flags=0x0 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7311feb640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-56 to cache LRU end 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7311feb640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-56 : miss 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.356+0000 7f7311feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7311feb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7311feb640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-56 info.flags=0x0 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7311feb640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-56 to cache LRU end 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7311feb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16): adding source pipe: {s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7311feb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16): adding dest pipe: {s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7311feb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7311feb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7311feb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2277286c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2277286c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2277286c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2277286c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768a1a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 20 run: stack=0x55c22768aee0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227727f70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227727f70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.357 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 53638903c22d9d5a515924324a6ce9ba49cc916ab5c115726a35d231346bd820 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:53638903c22d9d5a515924324a6ce9ba49cc916ab5c115726a35d231346bd820 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 46aec5c3f5cc14d1932d3a203179e3f5ca9a3a9ec08e5877f4465ab2260db0e4 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=46aec5c3f5cc14d1932d3a203179e3f5ca9a3a9ec08e5877f4465ab2260db0e4 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=22, curl_handle=0x55c227273060 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f7363925900 20 run: stack=0x55c22767e040 is io blocked 2026-03-20T11:46:11.358 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.357+0000 7f72be7fc640 20 link_request req_data=0x55c22767dd80 req_data->id=22, curl_handle=0x55c227273060 2026-03-20T11:46:11.358 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.358+0000 7f3e4b07c640 -1 req 3315406984507172057 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.358 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.358+0000 7f3e97915640 -1 req 12196896906273532977 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.358 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.360+0000 7f3e7f8e5640 -1 req 16957631926683461140 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22774da60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22774da60:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768fd80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f7363925900 15 stack 0x55c227687d10 end 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f7363925900 20 run: stack=0x55c227687d10 is done 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f7363925900 20 collect(): s=0x55c22753d3c0 stack=0x55c227687d10 is complete 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f7363925900 15 stack 0x55c22753d3c0 end 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is done 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f7363925900 4 starting trim on bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f7363925900 20 enqueued request req=0x55c227686910 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f7363925900 20 req: 0x55c227686910 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is io blocked 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f73117ea640 20 dequeued request req=0x55c227686910 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f73117ea640 20 RGWWQ: empty 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f73117ea640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 : miss 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f73117ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f73117ea640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f73117ea640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 info.flags=0x17 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f73117ea640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 to cache LRU end 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f73117ea640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f73117ea640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f73117ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f73117ea640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-27 : miss 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f73117ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f73117ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f73117ea640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-27 info.flags=0x0 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f73117ea640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-27 to cache LRU end 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f73117ea640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-27 : miss 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f73117ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f73117ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f73117ea640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-27 info.flags=0x0 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f73117ea640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-27 to cache LRU end 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f73117ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32): adding source pipe: {s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f73117ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32): adding dest pipe: {s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.358+0000 7f73117ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f73117ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f73117ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768fd80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22774da60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22774da60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 468dd943ef55cdfbea5ae330305ce4eae3c2310698be7e27c2f3844570ceaaac 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:468dd943ef55cdfbea5ae330305ce4eae3c2310698be7e27c2f3844570ceaaac 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 4e6cb2af4ee2648562d719d709b46187c961b122998171f14671d4451102e432 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=4e6cb2af4ee2648562d719d709b46187c961b122998171f14671d4451102e432 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=23, curl_handle=0x55c2272606a0 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f7363925900 20 run: stack=0x55c227687d10 is io blocked 2026-03-20T11:46:11.360 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.359+0000 7f72be7fc640 20 link_request req_data=0x7f72f8006330 req_data->id=23, curl_handle=0x55c2272606a0 2026-03-20T11:46:11.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c2272718b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7363925900 15 stack 0x55c22767e740 end 2026-03-20T11:46:11.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7363925900 20 run: stack=0x55c22767e740 is done 2026-03-20T11:46:11.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7363925900 20 collect(): s=0x55c227677bf0 stack=0x55c22767e740 is complete 2026-03-20T11:46:11.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7363925900 15 stack 0x55c227677bf0 end 2026-03-20T11:46:11.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7363925900 20 run: stack=0x55c227677bf0 is done 2026-03-20T11:46:11.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7363925900 4 starting trim on bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:46:11.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7363925900 20 enqueued request req=0x55c22767f320 2026-03-20T11:46:11.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7363925900 20 req: 0x55c22767f320 2026-03-20T11:46:11.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7363925900 20 run: stack=0x55c227677bf0 is io blocked 2026-03-20T11:46:11.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7310fe9640 20 dequeued request req=0x55c22767f320 2026-03-20T11:46:11.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7310fe9640 20 RGWWQ: empty 2026-03-20T11:46:11.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7310fe9640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 : miss 2026-03-20T11:46:11.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.397+0000 7f7310fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.398+0000 7f7310fe9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:11.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.398+0000 7f7310fe9640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 info.flags=0x17 2026-03-20T11:46:11.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.398+0000 7f7310fe9640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 to cache LRU end 2026-03-20T11:46:11.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.398+0000 7f7310fe9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.398+0000 7f7310fe9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.398+0000 7f7310fe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:46:11.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.398+0000 7f7310fe9640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-44 : miss 2026-03-20T11:46:11.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.398+0000 7f7310fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.398+0000 7f7310fe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.398+0000 7f7310fe9640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-44 info.flags=0x0 2026-03-20T11:46:11.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.398+0000 7f7310fe9640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-44 to cache LRU end 2026-03-20T11:46:11.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.398+0000 7f7310fe9640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-44 : miss 2026-03-20T11:46:11.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.398+0000 7f7310fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.398+0000 7f7310fe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.398+0000 7f7310fe9640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-44 info.flags=0x0 2026-03-20T11:46:11.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.398+0000 7f7310fe9640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-44 to cache LRU end 2026-03-20T11:46:11.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.398+0000 7f7310fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4): adding source pipe: {s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.398+0000 7f7310fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4): adding dest pipe: {s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.398+0000 7f7310fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.398+0000 7f7310fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.398+0000 7f7310fe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:46:11.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.398+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.398+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.398+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.398+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.398+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.398+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c2272718b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 run: stack=0x55c227677bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768e0c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768e0c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227727f70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227727f70:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.397 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 361e8c1bf381d0fcc7216d8a1fedecdecdf823d6c85814e7e97a587ac56a6df4 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:361e8c1bf381d0fcc7216d8a1fedecdecdf823d6c85814e7e97a587ac56a6df4 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = ff22e39e68489c8769257bdced388f6bbe521d814419e9564867677626e83507 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=ff22e39e68489c8769257bdced388f6bbe521d814419e9564867677626e83507 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=24, curl_handle=0x55c227273060 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 run: stack=0x55c22767e740 is io blocked 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c2276945e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 15 stack 0x55c227260bc0 end 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 run: stack=0x55c227260bc0 is done 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768a1a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 15 stack 0x55c22767e040 end 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 run: stack=0x55c22767e040 is done 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 collect(): s=0x55c227688970 stack=0x55c227260bc0 is complete 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 15 stack 0x55c227688970 end 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f72be7fc640 20 link_request req_data=0x55c22767dd80 req_data->id=24, curl_handle=0x55c227273060 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 run: stack=0x55c227688970 is done 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 collect(): s=0x55c22768aee0 stack=0x55c22767e040 is complete 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 15 stack 0x55c22768aee0 end 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 run: stack=0x55c22768aee0 is done 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 4 starting trim on bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 4 starting trim on bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 enqueued request req=0x55c227685140 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 req: 0x55c227685140 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 run: stack=0x55c227688970 is io blocked 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227727f70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227727f70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 enqueued request req=0x55c227264e10 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 req: 0x55c227685140 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 req: 0x55c227264e10 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f7363925900 20 run: stack=0x55c22768aee0 is io blocked 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f73107e8640 20 dequeued request req=0x55c227685140 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f73107e8640 20 RGWWQ: 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f73107e8640 20 req: 0x55c227264e10 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f73107e8640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 : miss 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f73107e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f730ffe7640 20 dequeued request req=0x55c227264e10 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f730ffe7640 20 RGWWQ: empty 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f730ffe7640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 : miss 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f730ffe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f73107e8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f73107e8640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 info.flags=0x17 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f73107e8640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 to cache LRU end 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f73107e8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f73107e8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f73107e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f73107e8640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-17 : miss 2026-03-20T11:46:11.399 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.399+0000 7f73107e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.399 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.401+0000 7f3e46072640 -1 req 1911652751329186297 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.400 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f73107e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.400 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f73107e8640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-17 info.flags=0x0 2026-03-20T11:46:11.400 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f73107e8640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-17 to cache LRU end 2026-03-20T11:46:11.400 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f73107e8640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-17 : miss 2026-03-20T11:46:11.400 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f73107e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.400 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f730ffe7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:11.400 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f730ffe7640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 info.flags=0x17 2026-03-20T11:46:11.400 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f730ffe7640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 to cache LRU end 2026-03-20T11:46:11.400 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f730ffe7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.400 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f730ffe7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.400 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f730ffe7640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:46:11.400 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f730ffe7640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:46:11.400 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f730ffe7640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-46 : miss 2026-03-20T11:46:11.400 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f730ffe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.400 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f73107e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.400 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f73107e8640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-17 info.flags=0x0 2026-03-20T11:46:11.400 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f73107e8640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-17 to cache LRU end 2026-03-20T11:46:11.400 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f73107e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22): adding source pipe: {s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.400 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f73107e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22): adding dest pipe: {s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f73107e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f73107e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f73107e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c2276945e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 20 run: stack=0x55c227688970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768e0c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768e0c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = a7a4abc0c78deeac30744174aa8eb4e7fd2dafd7ef8aecbf1ceebf4c101e2fe9 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:a7a4abc0c78deeac30744174aa8eb4e7fd2dafd7ef8aecbf1ceebf4c101e2fe9 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 39fd10ec80d7bc3e429698d73a9ab8e01df73ac2deca95878436ebe17ce037c6 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=39fd10ec80d7bc3e429698d73a9ab8e01df73ac2deca95878436ebe17ce037c6 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=25, curl_handle=0x55c227264b20 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f7363925900 20 run: stack=0x55c22767e040 is io blocked 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f72be7fc640 20 link_request req_data=0x55c227265420 req_data->id=25, curl_handle=0x55c227264b20 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f730ffe7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f730ffe7640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-46 info.flags=0x0 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f730ffe7640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-46 to cache LRU end 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f730ffe7640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-46 : miss 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.400+0000 7f730ffe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f730ffe7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f730ffe7640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-46 info.flags=0x0 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f730ffe7640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-46 to cache LRU end 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f730ffe7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6): adding source pipe: {s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.401 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f730ffe7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6): adding dest pipe: {s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f730ffe7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f730ffe7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f730ffe7640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22774da60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22774da60:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768fd80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 15 stack 0x55c227687d10 end 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 run: stack=0x55c227687d10 is done 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227727f70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 collect(): s=0x55c22753d3c0 stack=0x55c227687d10 is complete 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 15 stack 0x55c22753d3c0 end 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is done 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227727f70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227727f70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 4 starting trim on bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227727f70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c2277286c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c2277286c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 enqueued request req=0x55c227264e10 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 req: 0x55c227264e10 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is io blocked 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768fd80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 run: stack=0x55c22768aee0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22774da60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22774da60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f730f7e6640 20 dequeued request req=0x55c227264e10 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f730f7e6640 20 RGWWQ: empty 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f730f7e6640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 : miss 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.402 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 907431ac7711da2cefcb8f8d659924812b4e8481ffaa451b4a450922b87c2fe2 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f730f7e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:907431ac7711da2cefcb8f8d659924812b4e8481ffaa451b4a450922b87c2fe2 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 1d78b9dc5ce737307754bf74abb39a1064a107af002d3c90e1d5a2e5a6af5131 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=1d78b9dc5ce737307754bf74abb39a1064a107af002d3c90e1d5a2e5a6af5131 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=26, curl_handle=0x55c2272606a0 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f7363925900 20 run: stack=0x55c227687d10 is io blocked 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f72be7fc640 20 link_request req_data=0x7f72f8006330 req_data->id=26, curl_handle=0x55c2272606a0 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f730f7e6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f730f7e6640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 info.flags=0x17 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f730f7e6640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 to cache LRU end 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f730f7e6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f730f7e6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f730f7e6640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f730f7e6640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-37 : miss 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.401+0000 7f730f7e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f730f7e6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f730f7e6640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-37 info.flags=0x0 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f730f7e6640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-37 to cache LRU end 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f730f7e6640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-37 : miss 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f730f7e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f730f7e6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f730f7e6640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-37 info.flags=0x0 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f730f7e6640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-37 to cache LRU end 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f730f7e6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42): adding source pipe: {s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f730f7e6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42): adding dest pipe: {s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f730f7e6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f730f7e6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f730f7e6640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c2277286c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c2277286c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c2277286c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c2277286c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768a1a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c227727f70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c227727f70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.403 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.404 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.404 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.404 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.404 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.404 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.404 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.404 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.404 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.404 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 3fc7d45c8a15a68216fce35ad17b2a5f6728eddef20a7d435aafb087a5b4893e 2026-03-20T11:46:11.404 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.404 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.404 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.404 INFO:teuthology.orchestra.run.vm07.stderr:3fc7d45c8a15a68216fce35ad17b2a5f6728eddef20a7d435aafb087a5b4893e 2026-03-20T11:46:11.404 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.404 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.404 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.404 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.404 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 3e881d3410e4bd9fac9bd62d48b5e56f7cdd9878c01875736d784f61e14ea977 2026-03-20T11:46:11.404 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=3e881d3410e4bd9fac9bd62d48b5e56f7cdd9878c01875736d784f61e14ea977 2026-03-20T11:46:11.404 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.404 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.404 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.404 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=27, curl_handle=0x55c227681f00 2026-03-20T11:46:11.404 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f7363925900 20 run: stack=0x55c227260bc0 is io blocked 2026-03-20T11:46:11.404 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.402+0000 7f72be7fc640 20 link_request req_data=0x55c227682650 req_data->id=27, curl_handle=0x55c227681f00 2026-03-20T11:46:11.404 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.403+0000 7f3e668b3640 -1 req 18200249401818407933 0.003000004s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.404 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.403+0000 7f3e19819640 -1 req 9845749910744244412 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.404 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.404+0000 7f3e26032640 -1 req 11327629581189510347 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.440 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.440 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.440 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c2272718b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.440 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f7363925900 15 stack 0x55c22767e740 end 2026-03-20T11:46:11.440 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f7363925900 20 run: stack=0x55c22767e740 is done 2026-03-20T11:46:11.440 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.440 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f7363925900 20 collect(): s=0x55c227677bf0 stack=0x55c22767e740 is complete 2026-03-20T11:46:11.440 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.440 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.440 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.440 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.440 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f7363925900 15 stack 0x55c227677bf0 end 2026-03-20T11:46:11.440 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.440 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f7363925900 20 run: stack=0x55c227677bf0 is done 2026-03-20T11:46:11.440 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.440 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.440 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.440 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.440 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f7363925900 4 starting trim on bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:46:11.440 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f7363925900 20 enqueued request req=0x55c22767f320 2026-03-20T11:46:11.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f7363925900 20 req: 0x55c22767f320 2026-03-20T11:46:11.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f7363925900 20 run: stack=0x55c227677bf0 is io blocked 2026-03-20T11:46:11.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f730efe5640 20 dequeued request req=0x55c22767f320 2026-03-20T11:46:11.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f730efe5640 20 RGWWQ: empty 2026-03-20T11:46:11.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f730efe5640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 : miss 2026-03-20T11:46:11.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.442+0000 7f730efe5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.443+0000 7f730efe5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:46:11.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.443+0000 7f730efe5640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 info.flags=0x17 2026-03-20T11:46:11.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.443+0000 7f730efe5640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 to cache LRU end 2026-03-20T11:46:11.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.443+0000 7f730efe5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.443+0000 7f730efe5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.443+0000 7f730efe5640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:46:11.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.443+0000 7f730efe5640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-1 : miss 2026-03-20T11:46:11.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.443+0000 7f730efe5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.443+0000 7f730efe5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.443+0000 7f730efe5640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-1 info.flags=0x0 2026-03-20T11:46:11.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.443+0000 7f730efe5640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-1 to cache LRU end 2026-03-20T11:46:11.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.443+0000 7f730efe5640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-1 : miss 2026-03-20T11:46:11.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.443+0000 7f730efe5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.443+0000 7f730efe5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.443+0000 7f730efe5640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-1 info.flags=0x0 2026-03-20T11:46:11.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.443+0000 7f730efe5640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-1 to cache LRU end 2026-03-20T11:46:11.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.443+0000 7f730efe5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1): adding source pipe: {s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.443+0000 7f730efe5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1): adding dest pipe: {s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.443+0000 7f730efe5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.443+0000 7f730efe5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.443+0000 7f730efe5640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:46:11.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c2272718b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 run: stack=0x55c227677bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 25bdd552bf2bb8ac86c08cd12d9ca2d472c09e27e293e104278487903264248b 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:25bdd552bf2bb8ac86c08cd12d9ca2d472c09e27e293e104278487903264248b 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 499ba6158bab9b4a824a02701c7e1435e09adf3dd2e1dc703051c0c9062f70c2 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=499ba6158bab9b4a824a02701c7e1435e09adf3dd2e1dc703051c0c9062f70c2 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=28, curl_handle=0x55c2272606a0 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 run: stack=0x55c22767e740 is io blocked 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768e0c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768e0c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f72be7fc640 20 link_request req_data=0x55c22767dd80 req_data->id=28, curl_handle=0x55c2272606a0 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22774da60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22774da60:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c2276945e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 15 stack 0x55c22767e040 end 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 run: stack=0x55c22767e040 is done 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768fd80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 15 stack 0x55c227687d10 end 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 run: stack=0x55c227687d10 is done 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 collect(): s=0x55c227688970 stack=0x55c22767e040 is complete 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 15 stack 0x55c227688970 end 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 run: stack=0x55c227688970 is done 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 collect(): s=0x55c22768aee0 stack=0x55c227687d10 is complete 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 15 stack 0x55c22768aee0 end 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 run: stack=0x55c22768aee0 is done 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 4 starting trim on bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 4 starting trim on bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 enqueued request req=0x55c227686910 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 req: 0x55c227686910 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 run: stack=0x55c227688970 is io blocked 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 enqueued request req=0x55c227685140 2026-03-20T11:46:11.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 req: 0x55c227686910 2026-03-20T11:46:11.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 req: 0x55c227685140 2026-03-20T11:46:11.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f7363925900 20 run: stack=0x55c22768aee0 is io blocked 2026-03-20T11:46:11.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f730e7e4640 20 dequeued request req=0x55c227686910 2026-03-20T11:46:11.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f730e7e4640 20 RGWWQ: 2026-03-20T11:46:11.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f730e7e4640 20 req: 0x55c227685140 2026-03-20T11:46:11.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f731bfff640 20 dequeued request req=0x55c227685140 2026-03-20T11:46:11.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f731bfff640 20 RGWWQ: empty 2026-03-20T11:46:11.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f731bfff640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 : miss 2026-03-20T11:46:11.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f730e7e4640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 : miss 2026-03-20T11:46:11.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f731bfff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f730e7e4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.445 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.447+0000 7f3e4486f640 -1 req 16119933433790592604 0.003000004s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f731bfff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f731bfff640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 info.flags=0x17 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f731bfff640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 to cache LRU end 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f731bfff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f731bfff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f731bfff640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f731bfff640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f731bfff640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-48 : miss 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f731bfff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f730e7e4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f730e7e4640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 info.flags=0x17 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f730e7e4640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 to cache LRU end 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f730e7e4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f730e7e4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f730e7e4640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f730e7e4640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-7 : miss 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.444+0000 7f730e7e4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f731bfff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f731bfff640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-48 info.flags=0x0 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f731bfff640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-48 to cache LRU end 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f731bfff640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-48 : miss 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f731bfff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c227727f70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c227727f70:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768a1a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 15 stack 0x55c227260bc0 end 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 run: stack=0x55c227260bc0 is done 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 collect(): s=0x55c22753d3c0 stack=0x55c227260bc0 is complete 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 15 stack 0x55c22753d3c0 end 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is done 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 4 starting trim on bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 enqueued request req=0x55c227264e10 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 req: 0x55c227264e10 2026-03-20T11:46:11.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f73267fc640 20 dequeued request req=0x55c227264e10 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f73267fc640 20 RGWWQ: empty 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f73267fc640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 : miss 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f73267fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is io blocked 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f730e7e4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f730e7e4640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-7 info.flags=0x0 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f730e7e4640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-7 to cache LRU end 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f730e7e4640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-7 : miss 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f730e7e4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f731bfff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f731bfff640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-48 info.flags=0x0 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f731bfff640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-48 to cache LRU end 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f731bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8): adding source pipe: {s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f731bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8): adding dest pipe: {s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f731bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f731bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f731bfff640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c2276945e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 run: stack=0x55c22768aee0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768fd80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768fd80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.448 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = f8cfe6944eebf7b6057a35fdb7096553a6052bc6e177533824d9d542adffb7cf 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:f8cfe6944eebf7b6057a35fdb7096553a6052bc6e177533824d9d542adffb7cf 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = b033f9d5b416eb0f9e2f98f553b16a5b5a20c83c47f173d0cc690a46dd5d2935 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b033f9d5b416eb0f9e2f98f553b16a5b5a20c83c47f173d0cc690a46dd5d2935 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=29, curl_handle=0x55c227681f00 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 run: stack=0x55c227260bc0 is io blocked 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f72be7fc640 20 link_request req_data=0x55c227682650 req_data->id=29, curl_handle=0x55c227681f00 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f730e7e4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f730e7e4640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-7 info.flags=0x0 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f730e7e4640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-7 to cache LRU end 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f730e7e4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12): adding source pipe: {s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f730e7e4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12): adding dest pipe: {s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f730e7e4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f730e7e4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f730e7e4640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768e0c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 run: stack=0x55c227688970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768a1a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768a1a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = b9d9892ab05fc2990cfd15858108694192ec5cbb9cb6481f1db3d05f329ec373 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.449 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:b9d9892ab05fc2990cfd15858108694192ec5cbb9cb6481f1db3d05f329ec373 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 3b34fcb112eb4eeb272d4b060854273d10a25a50269fee0101388f8eb153e332 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=3b34fcb112eb4eeb272d4b060854273d10a25a50269fee0101388f8eb153e332 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=30, curl_handle=0x55c227264b20 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.445+0000 7f7363925900 20 run: stack=0x55c227687d10 is io blocked 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.446+0000 7f72be7fc640 20 link_request req_data=0x7f72f8006330 req_data->id=30, curl_handle=0x55c227264b20 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.446+0000 7f73267fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.446+0000 7f73267fc640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 info.flags=0x17 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.446+0000 7f73267fc640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 to cache LRU end 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.446+0000 7f73267fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.446+0000 7f73267fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.446+0000 7f73267fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.446+0000 7f73267fc640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-59 : miss 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.446+0000 7f73267fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.446+0000 7f73267fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.446+0000 7f73267fc640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-59 info.flags=0x0 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.446+0000 7f73267fc640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-59 to cache LRU end 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.446+0000 7f73267fc640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-59 : miss 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.446+0000 7f73267fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f73267fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f73267fc640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-59 info.flags=0x0 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f73267fc640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-59 to cache LRU end 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f73267fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19): adding source pipe: {s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f73267fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19): adding dest pipe: {s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f73267fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f73267fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f73267fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c2277287a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22774da60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22774da60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.450 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.451 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 1f533d745c3fc5192d011cedbb0cc9c5befc17d1a27fda8751e6024551ce2df5 2026-03-20T11:46:11.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.451 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.451 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.451 INFO:teuthology.orchestra.run.vm07.stderr:1f533d745c3fc5192d011cedbb0cc9c5befc17d1a27fda8751e6024551ce2df5 2026-03-20T11:46:11.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 2bad5920bfc338d7f787b0aa52008fa0bb8f469fc0b72c3da04edd3899cb9173 2026-03-20T11:46:11.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2bad5920bfc338d7f787b0aa52008fa0bb8f469fc0b72c3da04edd3899cb9173 2026-03-20T11:46:11.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=31, curl_handle=0x55c227273060 2026-03-20T11:46:11.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f7363925900 20 run: stack=0x55c22767e040 is io blocked 2026-03-20T11:46:11.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.447+0000 7f72be7fc640 20 link_request req_data=0x55c227265420 req_data->id=31, curl_handle=0x55c227273060 2026-03-20T11:46:11.451 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.447+0000 7f3e5308c640 -1 req 12141468186545342594 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.451 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.448+0000 7f3e7d0e0640 -1 req 12779582155622552441 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.451 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.449+0000 7f3e1d821640 -1 req 17210681972558069936 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.486 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.486 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.486 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c2272718b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.486 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 15 stack 0x55c22767e740 end 2026-03-20T11:46:11.486 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 run: stack=0x55c22767e740 is done 2026-03-20T11:46:11.486 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768fd80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.486 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768fd80:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.486 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.486 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 collect(): s=0x55c227677bf0 stack=0x55c22767e740 is complete 2026-03-20T11:46:11.486 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.486 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.486 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.486 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.486 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 15 stack 0x55c227677bf0 end 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 run: stack=0x55c227677bf0 is done 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c2276945e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 15 stack 0x55c227260bc0 end 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 run: stack=0x55c227260bc0 is done 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 collect(): s=0x55c22768aee0 stack=0x55c227260bc0 is complete 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 15 stack 0x55c22768aee0 end 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 run: stack=0x55c22768aee0 is done 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 4 starting trim on bucket=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 4 starting trim on bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 enqueued request req=0x55c22726fe00 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 req: 0x55c22726fe00 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7324ff9640 20 dequeued request req=0x55c22726fe00 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7324ff9640 20 RGWWQ: empty 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7324ff9640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 : miss 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7324ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 run: stack=0x55c227677bf0 is io blocked 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 enqueued request req=0x55c22767f320 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 req: 0x55c22767f320 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f73177f6640 20 dequeued request req=0x55c22767f320 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f73177f6640 20 RGWWQ: empty 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f73177f6640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 : miss 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f73177f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.487 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7363925900 20 run: stack=0x55c22768aee0 is io blocked 2026-03-20T11:46:11.489 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7324ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:46:11.489 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7324ff9640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 info.flags=0x17 2026-03-20T11:46:11.489 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7324ff9640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 to cache LRU end 2026-03-20T11:46:11.489 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7324ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.489 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.488+0000 7f7324ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.489 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7324ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 2026-03-20T11:46:11.489 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7324ff9640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-6 : miss 2026-03-20T11:46:11.489 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7324ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.489 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f73177f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.489 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f73177f6640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 info.flags=0x17 2026-03-20T11:46:11.489 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f73177f6640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 to cache LRU end 2026-03-20T11:46:11.489 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f73177f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.489 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f73177f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.489 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f73177f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:46:11.489 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f73177f6640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-21 : miss 2026-03-20T11:46:11.489 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f73177f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.489 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768a1a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.489 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768a1a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.489 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768e0c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.489 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 15 stack 0x55c227687d10 end 2026-03-20T11:46:11.489 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 run: stack=0x55c227687d10 is done 2026-03-20T11:46:11.489 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22774da60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.489 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22774da60:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.489 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.489 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 collect(): s=0x55c227688970 stack=0x55c227687d10 is complete 2026-03-20T11:46:11.489 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 15 stack 0x55c227688970 end 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 run: stack=0x55c227688970 is done 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c2277287a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 15 stack 0x55c22767e040 end 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 run: stack=0x55c22767e040 is done 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f73177f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f73177f6640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-21 info.flags=0x0 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f73177f6640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-21 to cache LRU end 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f73177f6640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-21 : miss 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f73177f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7324ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7324ff9640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-6 info.flags=0x0 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7324ff9640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-6 to cache LRU end 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7324ff9640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-6 : miss 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7324ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 collect(): s=0x55c22753d3c0 stack=0x55c22767e040 is complete 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c22753d3c0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 15 stack 0x55c22753d3c0 end 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 run: stack=0x55c22753d3c0 is done 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 4 starting trim on bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 4 starting trim on bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 enqueued request req=0x55c2276742c0 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 req: 0x55c2276742c0 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 run: stack=0x55c227688970 is io blocked 2026-03-20T11:46:11.490 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 enqueued request req=0x55c227686910 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 req: 0x55c2276742c0 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 req: 0x55c227686910 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 run: stack=0x55c22767e040 is io blocked 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f731b7fe640 20 dequeued request req=0x55c2276742c0 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f731b7fe640 20 RGWWQ: 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f731b7fe640 20 req: 0x55c227686910 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7325ffb640 20 dequeued request req=0x55c227686910 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7325ffb640 20 RGWWQ: empty 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f731b7fe640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 : miss 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7325ffb640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 : miss 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f731b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7325ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7324ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7324ff9640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-6 info.flags=0x0 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7324ff9640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-6 to cache LRU end 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7324ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11): adding source pipe: {s={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7324ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11): adding dest pipe: {s={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7324ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7324ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7324ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f73177f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f73177f6640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-21 info.flags=0x0 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f73177f6640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-21 to cache LRU end 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f73177f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26): adding source pipe: {s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f73177f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26): adding dest pipe: {s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f73177f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f73177f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f73177f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22774da60:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 run: stack=0x55c227677bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768e0c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.491 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768a1a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 run: stack=0x55c22768aee0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768e0c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.489+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f731b7fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f731b7fe640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 info.flags=0x17 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f731b7fe640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 to cache LRU end 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f731b7fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f731b7fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f731b7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f731b7fe640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-41 : miss 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f731b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 3b26ff4a63031aa4424f81f20072365e1c809ab3225568264e779579c4e29278 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:3b26ff4a63031aa4424f81f20072365e1c809ab3225568264e779579c4e29278 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 397d58d1e1c7c039233c47a0c76fda50d357764004d890b3ee81a10c12779313 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=397d58d1e1c7c039233c47a0c76fda50d357764004d890b3ee81a10c12779313 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=32, curl_handle=0x55c227273060 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 run: stack=0x55c227687d10 is io blocked 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768fd80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768fd80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 733485bfb818d50b2e1cab1e6a56170ecb98c2d51f9f9f6dbe6ec9d8efba5dfe 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:733485bfb818d50b2e1cab1e6a56170ecb98c2d51f9f9f6dbe6ec9d8efba5dfe 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 7e3d0f0bf801124b273e73315ec31ed5ac1b7fefa2824e72c4e2ddb5284697a5 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=7e3d0f0bf801124b273e73315ec31ed5ac1b7fefa2824e72c4e2ddb5284697a5 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=33, curl_handle=0x55c227264b20 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 run: stack=0x55c227260bc0 is io blocked 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f72be7fc640 20 link_request req_data=0x55c227265420 req_data->id=32, curl_handle=0x55c227273060 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f72be7fc640 20 link_request req_data=0x7f72f8006330 req_data->id=33, curl_handle=0x55c227264b20 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7325ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7325ffb640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 info.flags=0x17 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7325ffb640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 to cache LRU end 2026-03-20T11:46:11.493 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7325ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7325ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7325ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7325ffb640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-49 : miss 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7325ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f731b7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f731b7fe640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-41 info.flags=0x0 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f731b7fe640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-41 to cache LRU end 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f731b7fe640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-41 : miss 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f731b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7325ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7325ffb640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-49 info.flags=0x0 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7325ffb640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-49 to cache LRU end 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7325ffb640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-49 : miss 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7325ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f731b7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f731b7fe640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-41 info.flags=0x0 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f731b7fe640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-41 to cache LRU end 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f731b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1): adding source pipe: {s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f731b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1): adding dest pipe: {s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f731b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f731b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f731b7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768cce0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 run: stack=0x55c227688970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c2272718b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c2272718b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = d37e22fffbed4db87eec2afa0017bc68f690a053a8e67eb6cfc943417d33f554 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr:d37e22fffbed4db87eec2afa0017bc68f690a053a8e67eb6cfc943417d33f554 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = fa7c423811204790646444c0addc2f260125aabdf72ab916ff596ab659072539 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=fa7c423811204790646444c0addc2f260125aabdf72ab916ff596ab659072539 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=34, curl_handle=0x55c227681f00 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f7363925900 20 run: stack=0x55c22767e740 is io blocked 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.490+0000 7f72be7fc640 20 link_request req_data=0x55c227682650 req_data->id=34, curl_handle=0x55c227681f00 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7325ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7325ffb640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-49 info.flags=0x0 2026-03-20T11:46:11.495 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7325ffb640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-49 to cache LRU end 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7325ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9): adding source pipe: {s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7325ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9): adding dest pipe: {s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7325ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7325ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7325ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768f2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c2277287a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 20 run: stack=0x55c22767e040 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c22768f2f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c22768f2f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = d0cac8ac8a9200d5a0e87f1f44674faf8ef8ccc690fcff0197129c50b81bfb39 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:d0cac8ac8a9200d5a0e87f1f44674faf8ef8ccc690fcff0197129c50b81bfb39 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.497 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 8ebf11db62f135d41af9ac799aaeb78926453ee6e0bd23742c697dcd7d03a782 2026-03-20T11:46:11.497 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=8ebf11db62f135d41af9ac799aaeb78926453ee6e0bd23742c697dcd7d03a782 2026-03-20T11:46:11.497 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.497 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.497 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.497 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=35, curl_handle=0x55c2272606a0 2026-03-20T11:46:11.497 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f7363925900 20 run: stack=0x55c227685830 is io blocked 2026-03-20T11:46:11.497 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.491+0000 7f72be7fc640 20 link_request req_data=0x55c22767dd80 req_data->id=35, curl_handle=0x55c2272606a0 2026-03-20T11:46:11.497 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.492+0000 7f3e828eb640 -1 req 1868979784857521538 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.497 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.492+0000 7f3e1c01e640 -1 req 18218875308908553829 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.497 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.493+0000 7f3eef9c5640 -1 req 2946024283560888447 0.003000005s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.497 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.493+0000 7f3e618a9640 -1 req 2341390744203205092 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.531 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768e0c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.531 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768e0c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.531 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22774da60:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.531 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 15 stack 0x55c227687d10 end 2026-03-20T11:46:11.531 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 run: stack=0x55c227687d10 is done 2026-03-20T11:46:11.531 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c2272718b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.531 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c2272718b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.531 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.531 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 collect(): s=0x55c227677bf0 stack=0x55c227687d10 is complete 2026-03-20T11:46:11.531 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.531 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.531 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.531 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.531 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 15 stack 0x55c227677bf0 end 2026-03-20T11:46:11.531 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.531 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 run: stack=0x55c227677bf0 is done 2026-03-20T11:46:11.531 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768cce0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.531 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 15 stack 0x55c22767e740 end 2026-03-20T11:46:11.531 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 run: stack=0x55c22767e740 is done 2026-03-20T11:46:11.531 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.531 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.531 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.531 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 collect(): s=0x55c227688970 stack=0x55c22767e740 is complete 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 15 stack 0x55c227688970 end 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 run: stack=0x55c227688970 is done 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 4 starting trim on bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22774da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 4 starting trim on bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22774da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 enqueued request req=0x55c22725f800 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 req: 0x55c22725f800 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 run: stack=0x55c227677bf0 is io blocked 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227727f70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227727f70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 enqueued request req=0x55c227685140 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 req: 0x55c22725f800 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 req: 0x55c227685140 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f7363925900 20 run: stack=0x55c227688970 is io blocked 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f73257fa640 20 dequeued request req=0x55c22725f800 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f73257fa640 20 RGWWQ: 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f73257fa640 20 req: 0x55c227685140 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f73257fa640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 : miss 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f73257fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f731affd640 20 dequeued request req=0x55c227685140 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f731affd640 20 RGWWQ: empty 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f731affd640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 : miss 2026-03-20T11:46:11.532 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f731affd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.533 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f73257fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:11.533 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f731affd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.533 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f73257fa640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 info.flags=0x17 2026-03-20T11:46:11.533 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f73257fa640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 to cache LRU end 2026-03-20T11:46:11.533 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f73257fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.533 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f73257fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.533 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f73257fa640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:46:11.533 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f731affd640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 info.flags=0x17 2026-03-20T11:46:11.533 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f731affd640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 to cache LRU end 2026-03-20T11:46:11.533 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f731affd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.533 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f731affd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.533 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f731affd640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:46:11.533 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.533+0000 7f73257fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:46:11.533 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f731affd640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:46:11.533 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f731affd640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-33 : miss 2026-03-20T11:46:11.533 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f73257fa640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-45 : miss 2026-03-20T11:46:11.533 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f73257fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.533 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f731affd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.533 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768fd80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.533 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768fd80:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768a1a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 15 stack 0x55c227260bc0 end 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 run: stack=0x55c227260bc0 is done 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c22768f2f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c22768f2f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 collect(): s=0x55c22768aee0 stack=0x55c227260bc0 is complete 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 15 stack 0x55c22768aee0 end 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 run: stack=0x55c22768aee0 is done 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c2277287a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 15 stack 0x55c227685830 end 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 run: stack=0x55c227685830 is done 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 collect(): s=0x55c22767e040 stack=0x55c227685830 is complete 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 15 stack 0x55c22767e040 end 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 run: stack=0x55c22767e040 is done 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 4 starting trim on bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2277286c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 4 starting trim on bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2277286c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 enqueued request req=0x55c22767f320 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 req: 0x55c22767f320 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 run: stack=0x55c22768aee0 is io blocked 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 enqueued request req=0x55c22726fe00 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 req: 0x55c22767f320 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 req: 0x55c22726fe00 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 run: stack=0x55c22767e040 is io blocked 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7319ffb640 20 dequeued request req=0x55c22767f320 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7319ffb640 20 RGWWQ: 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7319ffb640 20 req: 0x55c22726fe00 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7319ffb640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 : miss 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f731a7fc640 20 dequeued request req=0x55c22726fe00 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f731a7fc640 20 RGWWQ: empty 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f731a7fc640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 : miss 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f731a7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7319ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f73257fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f73257fa640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-45 info.flags=0x0 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f73257fa640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-45 to cache LRU end 2026-03-20T11:46:11.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f73257fa640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-45 : miss 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f73257fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f731affd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f731affd640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-33 info.flags=0x0 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f731affd640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-33 to cache LRU end 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f731affd640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-33 : miss 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f731affd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7319ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f731affd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7319ffb640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 info.flags=0x17 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f73257fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7319ffb640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 to cache LRU end 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7319ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7319ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7319ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f731affd640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-33 info.flags=0x0 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f731affd640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-33 to cache LRU end 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f731affd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38): adding source pipe: {s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f731affd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38): adding dest pipe: {s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f731affd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f73257fa640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-45 info.flags=0x0 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f73257fa640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-45 to cache LRU end 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f73257fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5): adding source pipe: {s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f73257fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5): adding dest pipe: {s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7319ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7319ffb640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-47 : miss 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f731affd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7319ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f73257fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f73257fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f731affd640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f73257fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227727f70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227727f70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22774da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227727f70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22774da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227727f70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f731a7fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22774da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f731a7fc640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 info.flags=0x17 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f731a7fc640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 to cache LRU end 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f731a7fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f731a7fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22774da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c22768fd80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f731a7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 run: stack=0x55c227688970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c22768cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768e0c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f731a7fc640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-58 : miss 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 run: stack=0x55c227677bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c22768cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f731a7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.534+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 6068c87ca1202e7bd4282750fa53b19b18a94785f09287f2ee578d445210e710 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:6068c87ca1202e7bd4282750fa53b19b18a94785f09287f2ee578d445210e710 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = e44598cff8897e0afb27ce26ae090eeba76af26a6b2b42f9ea90f83a8dd8d565 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=e44598cff8897e0afb27ce26ae090eeba76af26a6b2b42f9ea90f83a8dd8d565 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=36, curl_handle=0x55c2272606a0 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 run: stack=0x55c227685830 is io blocked 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c2272718b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c2272718b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 01457d76449febf28f2f7c22fd4d16721dbd61e84f326405211911c3037f3b85 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.536 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:01457d76449febf28f2f7c22fd4d16721dbd61e84f326405211911c3037f3b85 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 74e833d15c7b58b6489c09fcf3a585120627ac69a9b8195424912dd157df4485 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=74e833d15c7b58b6489c09fcf3a585120627ac69a9b8195424912dd157df4485 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=37, curl_handle=0x55c227264b20 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 run: stack=0x55c227260bc0 is io blocked 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f72be7fc640 20 link_request req_data=0x55c22767dd80 req_data->id=36, curl_handle=0x55c2272606a0 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f72be7fc640 20 link_request req_data=0x7f72f8006330 req_data->id=37, curl_handle=0x55c227264b20 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7319ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7319ffb640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-47 info.flags=0x0 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7319ffb640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-47 to cache LRU end 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7319ffb640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-47 : miss 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7319ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7319ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7319ffb640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-47 info.flags=0x0 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7319ffb640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-47 to cache LRU end 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7319ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7): adding source pipe: {s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7319ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7): adding dest pipe: {s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7319ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7319ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7319ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f731a7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f731a7fc640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-58 info.flags=0x0 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f731a7fc640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-58 to cache LRU end 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f731a7fc640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-58 : miss 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f731a7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2277286c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2277286c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2277286c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2277286c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22774da60:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 run: stack=0x55c22768aee0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768f2f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768f2f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.538 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.537+0000 7f3ed318c640 -1 req 2351228061523905668 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.538 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.537+0000 7f3eec9bf640 -1 req 6267277586794169736 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.538 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.537+0000 7f3ef29cb640 -1 req 2308553480131900253 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.538 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.537+0000 7f3ecc17e640 -1 req 6872932655609904714 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = d815a537564088e680ea1f24c9917a61d148ba2077fa6c6b468213f52d856100 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:d815a537564088e680ea1f24c9917a61d148ba2077fa6c6b468213f52d856100 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 4aefbb683e04d447179c5bc743320f12a865bdbb711e0485952ae66b9a7b244b 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=4aefbb683e04d447179c5bc743320f12a865bdbb711e0485952ae66b9a7b244b 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=38, curl_handle=0x55c227681f00 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 run: stack=0x55c22767e740 is io blocked 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f72be7fc640 20 link_request req_data=0x55c227682650 req_data->id=38, curl_handle=0x55c227681f00 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f731a7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f731a7fc640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-58 info.flags=0x0 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f731a7fc640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-58 to cache LRU end 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f731a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18): adding source pipe: {s={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f731a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18): adding dest pipe: {s={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f731a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f731a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f731a7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768a1a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 run: stack=0x55c22767e040 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c227727f70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c227727f70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.535+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.536+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 91a13b46944864d14044f9249d8c9f35e42572df2da13cb02da780b6b0a46d31 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.536+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:91a13b46944864d14044f9249d8c9f35e42572df2da13cb02da780b6b0a46d31 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.536+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.536+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.536+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.536+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.536+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = b1f40c71005fe6ca7d460e69a767ca0bfe0613897080d3795f19e07ba1d21d9b 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.536+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b1f40c71005fe6ca7d460e69a767ca0bfe0613897080d3795f19e07ba1d21d9b 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.536+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.536+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.536+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.536+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=39, curl_handle=0x55c227273060 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.536+0000 7f7363925900 20 run: stack=0x55c227687d10 is io blocked 2026-03-20T11:46:11.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.536+0000 7f72be7fc640 20 link_request req_data=0x55c227265420 req_data->id=39, curl_handle=0x55c227273060 2026-03-20T11:46:11.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c22768cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c22768cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c22768fd80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 15 stack 0x55c227685830 end 2026-03-20T11:46:11.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 run: stack=0x55c227685830 is done 2026-03-20T11:46:11.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c2272718b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c2272718b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768f2f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768f2f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c227727f70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c227727f70:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 collect(): s=0x55c227688970 stack=0x55c227685830 is complete 2026-03-20T11:46:11.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 15 stack 0x55c227688970 end 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 run: stack=0x55c227688970 is done 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768e0c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 15 stack 0x55c227260bc0 end 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 run: stack=0x55c227260bc0 is done 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22774da60:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 15 stack 0x55c22767e740 end 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 run: stack=0x55c22767e740 is done 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768a1a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 15 stack 0x55c227687d10 end 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 run: stack=0x55c227687d10 is done 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 collect(): s=0x55c227677bf0 stack=0x55c227260bc0 is complete 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 15 stack 0x55c227677bf0 end 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 run: stack=0x55c227677bf0 is done 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 collect(): s=0x55c22768aee0 stack=0x55c22767e740 is complete 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 15 stack 0x55c22768aee0 end 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 run: stack=0x55c22768aee0 is done 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 collect(): s=0x55c22767e040 stack=0x55c227687d10 is complete 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 15 stack 0x55c22767e040 end 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 run: stack=0x55c22767e040 is done 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 4 starting trim on bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 4 starting trim on bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 enqueued request req=0x55c2276742c0 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 req: 0x55c2276742c0 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 run: stack=0x55c227688970 is io blocked 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22774da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 4 starting trim on bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22774da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f73197fa640 20 dequeued request req=0x55c2276742c0 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f73197fa640 20 RGWWQ: empty 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 enqueued request req=0x55c22725f800 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 req: 0x55c22725f800 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 run: stack=0x55c227677bf0 is io blocked 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227688f50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 4 starting trim on bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f73197fa640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 : miss 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227688f50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 enqueued request req=0x55c227264e10 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 req: 0x55c22725f800 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 req: 0x55c227264e10 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 run: stack=0x55c22768aee0 is io blocked 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 enqueued request req=0x55c227731010 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 req: 0x55c22725f800 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 req: 0x55c227264e10 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 req: 0x55c227731010 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7363925900 20 run: stack=0x55c22767e040 is io blocked 2026-03-20T11:46:11.579 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f73197fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7318ff9640 20 dequeued request req=0x55c22725f800 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7318ff9640 20 RGWWQ: 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7318ff9640 20 req: 0x55c227264e10 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7318ff9640 20 req: 0x55c227731010 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f73187f8640 20 dequeued request req=0x55c227264e10 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f73187f8640 20 RGWWQ: 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f73187f8640 20 req: 0x55c227731010 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7318ff9640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 : miss 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f73187f8640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 : miss 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7318ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f73187f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7317ff7640 20 dequeued request req=0x55c227731010 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7317ff7640 20 RGWWQ: empty 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7317ff7640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 : miss 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.578+0000 7f7317ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f7318ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f73187f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f7318ff9640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 info.flags=0x17 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f7318ff9640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 to cache LRU end 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f7318ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f7318ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f73187f8640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 info.flags=0x17 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f73187f8640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 to cache LRU end 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f73187f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f73187f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f7318ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f73187f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f7318ff9640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-20 : miss 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f73187f8640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-52 : miss 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f73187f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f7318ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f7317ff7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f7317ff7640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 info.flags=0x17 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f7317ff7640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 to cache LRU end 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f7317ff7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f7317ff7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f7317ff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f7317ff7640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-42 : miss 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f7317ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f73197fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f73197fa640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 info.flags=0x17 2026-03-20T11:46:11.580 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f73197fa640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 to cache LRU end 2026-03-20T11:46:11.581 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f73197fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.581 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f73197fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:11.581 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f73197fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:46:11.581 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f73197fa640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-30 : miss 2026-03-20T11:46:11.581 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f73197fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.581 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.583+0000 7f3e10807640 -1 req 5916598659156957349 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.581 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.583+0000 7f3ef71d4640 -1 req 7974797949518071798 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.581 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.583+0000 7f3e5a89b640 -1 req 4855037797635695098 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.582 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f7318ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.582 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f7318ff9640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-20 info.flags=0x0 2026-03-20T11:46:11.582 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f7318ff9640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-20 to cache LRU end 2026-03-20T11:46:11.582 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f7318ff9640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-20 : miss 2026-03-20T11:46:11.582 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f7318ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.582 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f73187f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.582 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f73187f8640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-52 info.flags=0x0 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f73187f8640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-52 to cache LRU end 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f73187f8640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-52 : miss 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f73187f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f7317ff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f7317ff7640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-42 info.flags=0x0 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f7317ff7640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-42 to cache LRU end 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f7317ff7640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-42 : miss 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.579+0000 7f7317ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7318ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7318ff9640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-20 info.flags=0x0 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7318ff9640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-20 to cache LRU end 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7318ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25): adding source pipe: {s={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7318ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25): adding dest pipe: {s={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7318ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7318ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7318ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7317ff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7317ff7640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-42 info.flags=0x0 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7317ff7640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-42 to cache LRU end 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7317ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2): adding source pipe: {s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7317ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2): adding dest pipe: {s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22774da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22774da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22774da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22774da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7317ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7317ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f73187f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f73197fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f73187f8640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-52 info.flags=0x0 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f73187f8640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-52 to cache LRU end 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7317ff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f73197fa640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-30 info.flags=0x0 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f73197fa640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-30 to cache LRU end 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f73197fa640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-30 : miss 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f73197fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f73187f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12): adding source pipe: {s={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f73187f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12): adding dest pipe: {s={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f73187f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f73187f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f73187f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768e0c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 run: stack=0x55c227677bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227688f50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768fd80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227688f50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768fd80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f73197fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f73197fa640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-30 info.flags=0x0 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f73197fa640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-30 to cache LRU end 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f73197fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35): adding source pipe: {s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f73197fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35): adding dest pipe: {s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f73197fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f73197fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f73197fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 31e271f4931decdc0b9b7395a6ecd54ace1bd3d147e548f670846fe2607c195a 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:31e271f4931decdc0b9b7395a6ecd54ace1bd3d147e548f670846fe2607c195a 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = ebce37fa4d8a72ea049f47253e227badcb70a38ca91599bc5a3f2c130c3ef180 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=ebce37fa4d8a72ea049f47253e227badcb70a38ca91599bc5a3f2c130c3ef180 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=40, curl_handle=0x55c227273060 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 run: stack=0x55c227687d10 is io blocked 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227688f50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f72be7fc640 20 link_request req_data=0x55c227265420 req_data->id=40, curl_handle=0x55c227273060 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227688f50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768a1a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 run: stack=0x55c22767e040 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768f2f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 run: stack=0x55c22768aee0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c2272718b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22774da60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.580+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.584 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = c84f7e4413b0dd728e42e300429cfb933cb52fc1c9b1e086cd3929f222cff64b 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:c84f7e4413b0dd728e42e300429cfb933cb52fc1c9b1e086cd3929f222cff64b 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = b0ba2775043f13218420f012063479861df55411c1ec83a26439ae9b20e71aad 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b0ba2775043f13218420f012063479861df55411c1ec83a26439ae9b20e71aad 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=41, curl_handle=0x55c227681f00 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f72be7fc640 20 link_request req_data=0x55c227682650 req_data->id=41, curl_handle=0x55c227681f00 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 run: stack=0x55c22767e740 is io blocked 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22774da60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 2406ea97abe3800623ebb146ac97d0b7db254bf600a49bdfdf522ef1d60136bc 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2406ea97abe3800623ebb146ac97d0b7db254bf600a49bdfdf522ef1d60136bc 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 7e2ec495d613df978b64d7778b951eb178df595dcac7f372c4dc4165ad5a60cb 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=7e2ec495d613df978b64d7778b951eb178df595dcac7f372c4dc4165ad5a60cb 2026-03-20T11:46:11.585 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=42, curl_handle=0x55c227264b20 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f72be7fc640 20 link_request req_data=0x7f72f8006330 req_data->id=42, curl_handle=0x55c227264b20 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 run: stack=0x55c227260bc0 is io blocked 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c227688f50:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 run: stack=0x55c227688970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c2272718b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c2272718b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 80b73f4e6d954057c5ebc61babf5b25181e8b404ae19d250a0c3b41216f5266a 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:80b73f4e6d954057c5ebc61babf5b25181e8b404ae19d250a0c3b41216f5266a 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = d54d543269ea772033ba99cbca1f411f32a8930add3ad5f642fc9dfe75ab8f99 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=d54d543269ea772033ba99cbca1f411f32a8930add3ad5f642fc9dfe75ab8f99 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=43, curl_handle=0x55c2272606a0 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f72be7fc640 20 link_request req_data=0x55c22767dd80 req_data->id=43, curl_handle=0x55c2272606a0 2026-03-20T11:46:11.586 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.581+0000 7f7363925900 20 run: stack=0x55c227685830 is io blocked 2026-03-20T11:46:11.586 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.583+0000 7f3ef39cd640 -1 req 7368170627789679735 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.622 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768fd80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768fd80:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768e0c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 15 stack 0x55c227687d10 end 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 run: stack=0x55c227687d10 is done 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22774da60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22774da60:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c2272718b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c2272718b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 collect(): s=0x55c227677bf0 stack=0x55c227687d10 is complete 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 15 stack 0x55c227677bf0 end 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 run: stack=0x55c227677bf0 is done 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768a1a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 15 stack 0x55c22767e740 end 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 run: stack=0x55c22767e740 is done 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768f2f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 15 stack 0x55c227260bc0 end 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 run: stack=0x55c227260bc0 is done 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c227688f50:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 15 stack 0x55c227685830 end 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 run: stack=0x55c227685830 is done 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 collect(): s=0x55c22767e040 stack=0x55c22767e740 is complete 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 15 stack 0x55c22767e040 end 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 run: stack=0x55c22767e040 is done 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 collect(): s=0x55c22768aee0 stack=0x55c227260bc0 is complete 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 15 stack 0x55c22768aee0 end 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 run: stack=0x55c22768aee0 is done 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 collect(): s=0x55c227688970 stack=0x55c227685830 is complete 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.623 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 15 stack 0x55c227688970 end 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 run: stack=0x55c227688970 is done 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 4 starting trim on bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 4 starting trim on bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 enqueued request req=0x55c227694790 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 req: 0x55c227694790 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 run: stack=0x55c227677bf0 is io blocked 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 4 starting trim on bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 enqueued request req=0x55c2276742c0 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 req: 0x55c227694790 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 req: 0x55c2276742c0 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 run: stack=0x55c227688970 is io blocked 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 4 starting trim on bucket=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 enqueued request req=0x55c22753d1d0 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 req: 0x55c227694790 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 req: 0x55c2276742c0 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 req: 0x55c22753d1d0 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 run: stack=0x55c22768aee0 is io blocked 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227729490:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227729490:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 enqueued request req=0x55c227264e10 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 req: 0x55c227694790 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 req: 0x55c2276742c0 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 req: 0x55c22753d1d0 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 req: 0x55c227264e10 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7363925900 20 run: stack=0x55c22767e040 is io blocked 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7315ff3640 20 dequeued request req=0x55c227694790 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7315ff3640 20 RGWWQ: 2026-03-20T11:46:11.624 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7315ff3640 20 req: 0x55c2276742c0 2026-03-20T11:46:11.625 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7315ff3640 20 req: 0x55c22753d1d0 2026-03-20T11:46:11.625 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7315ff3640 20 req: 0x55c227264e10 2026-03-20T11:46:11.625 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f73157f2640 20 dequeued request req=0x55c2276742c0 2026-03-20T11:46:11.625 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f73157f2640 20 RGWWQ: 2026-03-20T11:46:11.625 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f73157f2640 20 req: 0x55c22753d1d0 2026-03-20T11:46:11.625 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f73157f2640 20 req: 0x55c227264e10 2026-03-20T11:46:11.625 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7315ff3640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 : miss 2026-03-20T11:46:11.625 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f73157f2640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 : miss 2026-03-20T11:46:11.625 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f73157f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.625 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7315ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.625 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7316ff5640 20 dequeued request req=0x55c22753d1d0 2026-03-20T11:46:11.625 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7316ff5640 20 RGWWQ: 2026-03-20T11:46:11.625 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7316ff5640 20 req: 0x55c227264e10 2026-03-20T11:46:11.625 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7316ff5640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 : miss 2026-03-20T11:46:11.625 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f7316ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.625 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f73167f4640 20 dequeued request req=0x55c227264e10 2026-03-20T11:46:11.625 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f73167f4640 20 RGWWQ: empty 2026-03-20T11:46:11.625 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f73167f4640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 : miss 2026-03-20T11:46:11.625 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.624+0000 7f73167f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.626 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f73157f2640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.626 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f73157f2640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 info.flags=0x17 2026-03-20T11:46:11.626 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f73157f2640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 to cache LRU end 2026-03-20T11:46:11.626 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f73157f2640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.626 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f73157f2640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.626 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f73157f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:11.626 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f73157f2640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-57 : miss 2026-03-20T11:46:11.626 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f73157f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.626 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f7315ff3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:46:11.626 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f7315ff3640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 info.flags=0x17 2026-03-20T11:46:11.626 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f7315ff3640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 to cache LRU end 2026-03-20T11:46:11.626 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f7315ff3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.626 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f7315ff3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.626 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f7315ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:46:11.626 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f7315ff3640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-2 : miss 2026-03-20T11:46:11.626 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f7315ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.626 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f7316ff5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f7316ff5640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 info.flags=0x17 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f7316ff5640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 to cache LRU end 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f7316ff5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f7316ff5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f7316ff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f7316ff5640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-29 : miss 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f7316ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f73167f4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=387 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f73167f4640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 info.flags=0x17 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f73167f4640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 to cache LRU end 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f73167f4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f73167f4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=171 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f73167f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f73167f4640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.testx/tenanted-bucket : miss 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f73167f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f73157f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f73157f2640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-57 info.flags=0x0 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f73157f2640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-57 to cache LRU end 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f73157f2640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-57 : miss 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.625+0000 7f73157f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f73167f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f73167f4640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.testx/tenanted-bucket info.flags=0x0 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f73167f4640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.testx/tenanted-bucket to cache LRU end 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f73167f4640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.testx/tenanted-bucket : miss 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f73167f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7315ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7315ff3640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-2 info.flags=0x0 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7315ff3640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-2 to cache LRU end 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7315ff3640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-2 : miss 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7315ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f73157f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f73157f2640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-57 info.flags=0x0 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f73157f2640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-57 to cache LRU end 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f73157f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17): adding source pipe: {s={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f73157f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17): adding dest pipe: {s={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f73157f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f73157f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f73157f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7315ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7315ff3640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-2 info.flags=0x0 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7315ff3640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-2 to cache LRU end 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7315ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2): adding source pipe: {s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7315ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2): adding dest pipe: {s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774da60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c22768cce0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 run: stack=0x55c227688970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c22768e0c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c22768e0c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7315ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7315ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7315ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.627 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7316ff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7316ff5640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-29 info.flags=0x0 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7316ff5640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-29 to cache LRU end 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7316ff5640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-29 : miss 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7316ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f73167f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f73167f4640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.testx/tenanted-bucket info.flags=0x0 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f73167f4640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.testx/tenanted-bucket to cache LRU end 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f73167f4640 20 rgw async rados processor: reflect(): flow manager (bucket=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3): adding source pipe: {s={b=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f73167f4640 20 rgw async rados processor: reflect(): flow manager (bucket=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3): adding dest pipe: {s={b=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f73167f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f73167f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f73167f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = e611c7929fdace919ac2e91fd595f94aeaa8bc471a3afab985080e45094e331f 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:e611c7929fdace919ac2e91fd595f94aeaa8bc471a3afab985080e45094e331f 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 8c97211be63669a4b4e60ecdac3628bf237409f740e1849516ae582f61ced087 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=8c97211be63669a4b4e60ecdac3628bf237409f740e1849516ae582f61ced087 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=44, curl_handle=0x55c2272606a0 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7316ff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7316ff5640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-29 info.flags=0x0 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7316ff5640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-29 to cache LRU end 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f72be7fc640 20 link_request req_data=0x55c22767dd80 req_data->id=44, curl_handle=0x55c2272606a0 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7316ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34): adding source pipe: {s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7316ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34): adding dest pipe: {s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7316ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7316ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7316ff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 run: stack=0x55c227685830 is io blocked 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227729490:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227729490:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.628 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227729490:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c22768a1a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227729490:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c227680a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768fd80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 run: stack=0x55c227677bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768f2f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 run: stack=0x55c22767e040 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c2272718b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 run: stack=0x55c22768aee0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768a1a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22774da60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c227680a40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.626+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768a1a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 422e7d9988fa0a8a93d0c7f6d366779ac0edbae6baf1612a38d9685b6444a921 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:422e7d9988fa0a8a93d0c7f6d366779ac0edbae6baf1612a38d9685b6444a921 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 3efecef4e4f1f8402982a490738fba94de2441fcd286728bceec4302cc026073 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=3efecef4e4f1f8402982a490738fba94de2441fcd286728bceec4302cc026073 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=45, curl_handle=0x55c227264b20 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f72be7fc640 20 link_request req_data=0x7f72f8006330 req_data->id=45, curl_handle=0x55c227264b20 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 run: stack=0x55c227260bc0 is io blocked 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22774da60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.629 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = ae4131f5e743e05e9c62892748dd3d9458754f8a6d90c2029ab89d3b50bf3315 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:ae4131f5e743e05e9c62892748dd3d9458754f8a6d90c2029ab89d3b50bf3315 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 3df434e8f2e9d6a8ab8a4636af092556d44d36562ad7c09287a9aced5a722e39 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=3df434e8f2e9d6a8ab8a4636af092556d44d36562ad7c09287a9aced5a722e39 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=46, curl_handle=0x55c227681f00 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f72be7fc640 20 link_request req_data=0x55c227682650 req_data->id=46, curl_handle=0x55c227681f00 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 run: stack=0x55c22767e740 is io blocked 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c227680a40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 80d2e34452698a2b096b18a4cc7c2e7537c12e95b6534e1dc7f1b20ad24eb3b8 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:80d2e34452698a2b096b18a4cc7c2e7537c12e95b6534e1dc7f1b20ad24eb3b8 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 453f8fb12c60f55c2a5bb0b0024f2aea89d300110688410b1eec687cf53ffc79 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=453f8fb12c60f55c2a5bb0b0024f2aea89d300110688410b1eec687cf53ffc79 2026-03-20T11:46:11.630 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.631 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.631 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.631 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=47, curl_handle=0x55c227273060 2026-03-20T11:46:11.631 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f72be7fc640 20 link_request req_data=0x55c227265420 req_data->id=47, curl_handle=0x55c227273060 2026-03-20T11:46:11.631 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.627+0000 7f7363925900 20 run: stack=0x55c227687d10 is io blocked 2026-03-20T11:46:11.631 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.629+0000 7f3ef51d0640 -1 req 14073151436963564351 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.631 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.629+0000 7f3e5f0a4640 -1 req 13754457082572720492 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.631 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.629+0000 7f3e5c09e640 -1 req 10016662988505116943 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.631 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.630+0000 7f3e5388d640 -1 req 15175138281544871688 0.003000005s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.668 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c22768e0c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.668 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c22768e0c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 cr:s=0x55c227685830:op=0x55c22768cce0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 15 stack 0x55c227685830 end 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 run: stack=0x55c227685830 is done 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768a1a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768a1a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22774da60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22774da60:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 collect(): s=0x55c227688970 stack=0x55c227685830 is complete 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 15 stack 0x55c227688970 end 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 run: stack=0x55c227688970 is done 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 cr:s=0x55c227260bc0:op=0x55c22768fd80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 15 stack 0x55c227260bc0 end 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 run: stack=0x55c227260bc0 is done 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 cr:s=0x55c22767e740:op=0x55c22768f2f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 15 stack 0x55c22767e740 end 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 run: stack=0x55c22767e740 is done 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 collect(): s=0x55c227677bf0 stack=0x55c227260bc0 is complete 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 15 stack 0x55c227677bf0 end 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 run: stack=0x55c227677bf0 is done 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 collect(): s=0x55c22767e040 stack=0x55c22767e740 is complete 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22772fc50:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 15 stack 0x55c22767e040 end 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 run: stack=0x55c22767e040 is done 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 4 starting trim on bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.669 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.670 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227727f70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.670 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.670 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 4 starting trim on bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:46:11.670 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.670 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.670 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.670 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227727f70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.670 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 enqueued request req=0x55c227264e10 2026-03-20T11:46:11.670 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.670 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 req: 0x55c227264e10 2026-03-20T11:46:11.670 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 run: stack=0x55c227688970 is io blocked 2026-03-20T11:46:11.670 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.670 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.670 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7314ff1640 20 dequeued request req=0x55c227264e10 2026-03-20T11:46:11.670 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7314ff1640 20 RGWWQ: empty 2026-03-20T11:46:11.670 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 enqueued request req=0x55c22767f320 2026-03-20T11:46:11.670 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 RGWWQ: 2026-03-20T11:46:11.670 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 req: 0x55c22767f320 2026-03-20T11:46:11.670 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f73147f0640 20 dequeued request req=0x55c22767f320 2026-03-20T11:46:11.670 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f73147f0640 20 RGWWQ: empty 2026-03-20T11:46:11.670 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f73147f0640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 : miss 2026-03-20T11:46:11.670 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7314ff1640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 : miss 2026-03-20T11:46:11.670 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f73147f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.670 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7314ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.670 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f7363925900 20 run: stack=0x55c227677bf0 is io blocked 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f73147f0640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f73147f0640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 info.flags=0x17 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f73147f0640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 to cache LRU end 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f73147f0640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f73147f0640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f73147f0640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.670+0000 7f73147f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f73147f0640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-34 : miss 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f73147f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7314ff1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7314ff1640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 info.flags=0x17 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7314ff1640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 to cache LRU end 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7314ff1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7314ff1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7314ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7314ff1640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-23 : miss 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7314ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c227680a40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c227680a40:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c2272718b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7363925900 15 stack 0x55c227687d10 end 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7363925900 20 run: stack=0x55c227687d10 is done 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7363925900 20 collect(): s=0x55c22768aee0 stack=0x55c227687d10 is complete 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7363925900 20 cr:s=0x55c22768aee0:op=0x55c22768b3b0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7363925900 15 stack 0x55c22768aee0 end 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7363925900 20 run: stack=0x55c22768aee0 is done 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7314ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7314ff1640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-23 info.flags=0x0 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7314ff1640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-23 to cache LRU end 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7314ff1640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-23 : miss 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7314ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f73147f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f73147f0640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-34 info.flags=0x0 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f73147f0640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-34 to cache LRU end 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f73147f0640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-34 : miss 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f73147f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7314ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.671+0000 7f7314ff1640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-23 info.flags=0x0 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f73147f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:11.672 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7314ff1640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-23 to cache LRU end 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f73147f0640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-34 info.flags=0x0 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7314ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28): adding source pipe: {s={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f73147f0640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-34 to cache LRU end 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f73147f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39): adding source pipe: {s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7314ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28): adding dest pipe: {s={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f73147f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39): adding dest pipe: {s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7314ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7314ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f73147f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7314ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f73147f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f73147f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227727f70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227727f70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227727f70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c227727f70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c227262ef0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768c0a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 run: stack=0x55c227688970 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c227727f70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22772d990:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 run: stack=0x55c227677bf0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c227727f70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 2d9e96705d542d26a7a60466a4a21c7003eccf1f6c54d21748395775a4fb81a8 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2d9e96705d542d26a7a60466a4a21c7003eccf1f6c54d21748395775a4fb81a8 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = c29adeeb27d4ca9f210792c6f6c10d3c85ba60240020d93bb80579dd1db1d843 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c29adeeb27d4ca9f210792c6f6c10d3c85ba60240020d93bb80579dd1db1d843 2026-03-20T11:46:11.673 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=48, curl_handle=0x55c227273060 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f72be7fc640 20 link_request req_data=0x55c227265420 req_data->id=48, curl_handle=0x55c227273060 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 run: stack=0x55c227687d10 is io blocked 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227262ef0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227262ef0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.672+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.673+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.673+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.673+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:11 +0000 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114611Z 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.673+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = f92b4e7868a7e1c0fdebadf8f7106d152a79bae14fea1c9c6bd99509a9721706 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.673+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114611Z 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:f92b4e7868a7e1c0fdebadf8f7106d152a79bae14fea1c9c6bd99509a9721706 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.673+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.673+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.673+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.673+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.673+0000 7f7363925900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 80c063ca078cc43123d01f20e56dabf0343899e3c7ca73c781d32c5da4792df6 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.673+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=80c063ca078cc43123d01f20e56dabf0343899e3c7ca73c781d32c5da4792df6 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.673+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.673+0000 7f7363925900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114611Z 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.673+0000 7f7363925900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.673+0000 7f7363925900 20 register_request mgr=0x7ffd7ee0fc60 req_data->id=49, curl_handle=0x55c227681f00 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.673+0000 7f72be7fc640 20 link_request req_data=0x55c227682650 req_data->id=49, curl_handle=0x55c227681f00 2026-03-20T11:46:11.674 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.673+0000 7f7363925900 20 run: stack=0x55c22767e040 is io blocked 2026-03-20T11:46:11.674 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.675+0000 7f3ea8937640 -1 req 18301508506394499775 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.674 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:46:11.675+0000 7f3ea9939640 -1 req 12153827468268201746 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:46:11.714 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c227727f70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.714 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c227727f70:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.714 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 20 cr:s=0x55c227687d10:op=0x55c22768c0a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.714 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 15 stack 0x55c227687d10 end 2026-03-20T11:46:11.714 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 20 run: stack=0x55c227687d10 is done 2026-03-20T11:46:11.714 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227262ef0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:11.714 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c227262ef0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:11.714 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.714 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 20 collect(): s=0x55c227688970 stack=0x55c227687d10 is complete 2026-03-20T11:46:11.714 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.714 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.714 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.714 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 20 cr:s=0x55c227688970:op=0x55c22774ac30:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.714 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 15 stack 0x55c227688970 end 2026-03-20T11:46:11.714 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.714 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 20 run: stack=0x55c227688970 is done 2026-03-20T11:46:11.714 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 20 cr:s=0x55c22767e040:op=0x55c22772d990:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:11.715 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 15 stack 0x55c22767e040 end 2026-03-20T11:46:11.715 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 20 run: stack=0x55c22767e040 is done 2026-03-20T11:46:11.715 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.715 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.715 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 20 run: stack=0x55c227674780 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:11.715 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:11.715 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 20 collect(): s=0x55c227677bf0 stack=0x55c22767e040 is complete 2026-03-20T11:46:11.715 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 10 min_generation is 0 2026-03-20T11:46:11.715 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:11.715 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:11.715 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 20 cr:s=0x55c227677bf0:op=0x55c2276760d0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:11.715 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 15 stack 0x55c227677bf0 end 2026-03-20T11:46:11.715 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 20 stack->operate() returned ret=-22 2026-03-20T11:46:11.715 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 20 run: stack=0x55c227677bf0 is done 2026-03-20T11:46:11.715 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:11.715 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:11.715 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227678530:27BucketTrimInstanceCollectCR: operate() returned r=-22 2026-03-20T11:46:11.715 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c2276749c0:12BucketTrimCR: operate() 2026-03-20T11:46:11.715 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 20 writing bucket trim marker=12:ca472e1e:root::.bucket.meta.dipepu-54%3a4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14:head 2026-03-20T11:46:11.715 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227262ef0:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:11.715 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227262ef0:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:11.715 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.716+0000 7f7363925900 20 run: stack=0x55c227674780 is io blocked 2026-03-20T11:46:11.716 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.718+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227262ef0:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:11.716 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.718+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227262ef0:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:11.716 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.718+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227262ef0:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:11.716 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.718+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c227262ef0:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:11.716 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.718+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c2276749c0:12BucketTrimCR: operate() 2026-03-20T11:46:11.716 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.718+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c22772d990:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:11.717 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.718+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c22772d990:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:11.717 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.718+0000 7f7363925900 20 run: stack=0x55c227674780 is io blocked 2026-03-20T11:46:11.717 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.719+0000 7f7339ffb640 20 trim: bucket trim completed 2026-03-20T11:46:11.717 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.719+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c22772d990:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:11.717 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.719+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c22772d990:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:11.717 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.719+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c22772d990:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:11.717 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.719+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c22772d990:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:11.717 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.719+0000 7f7363925900 20 cr:s=0x55c227674780:op=0x55c2276749c0:12BucketTrimCR: operate() 2026-03-20T11:46:11.717 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.719+0000 7f7363925900 4 bucket index log processing completed in 0.745772183s 2026-03-20T11:46:11.717 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.719+0000 7f7363925900 15 stack 0x55c227674780 end 2026-03-20T11:46:11.717 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.719+0000 7f7363925900 20 run: stack=0x55c227674780 is done 2026-03-20T11:46:11.722 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.724+0000 7f7363925900 20 remove_watcher() i=5 2026-03-20T11:46:11.722 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.724+0000 7f7363925900 2 removed watcher, disabling cache 2026-03-20T11:46:11.724 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.725+0000 7f7363925900 20 remove_watcher() i=0 2026-03-20T11:46:11.724 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.725+0000 7f7363925900 20 remove_watcher() i=6 2026-03-20T11:46:11.724 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.726+0000 7f7363925900 20 remove_watcher() i=2 2026-03-20T11:46:11.724 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.726+0000 7f7363925900 20 remove_watcher() i=7 2026-03-20T11:46:11.724 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.726+0000 7f7363925900 20 remove_watcher() i=3 2026-03-20T11:46:11.724 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.726+0000 7f7363925900 20 remove_watcher() i=4 2026-03-20T11:46:11.724 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:11.726+0000 7f7363925900 20 remove_watcher() i=1 2026-03-20T11:46:16.338 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:16.339+0000 7f7629344640 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2026-03-20T11:46:16.338 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:16.339+0000 7f762d34c640 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2026-03-20T11:46:16.731 INFO:rgw_multi.tests:trimming on zone=b2 2026-03-20T11:46:16.731 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '50', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:46:16.731 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '50', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:46:16.731 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bilog autotrim --rgw-sync-log-trim-max-buckets 50 --rgw-zone b2 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 20 --debug-ms 0 2026-03-20T11:46:16.770 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:16.770 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:16.789 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.791+0000 7f339633c900 20 rados->read ofs=0 len=0 2026-03-20T11:46:16.789 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.791+0000 7f339633c900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:46:16.789 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.791+0000 7f339633c900 20 rados->read ofs=0 len=0 2026-03-20T11:46:16.790 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.792+0000 7f339633c900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:46:16.790 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.792+0000 7f339633c900 20 realm test-realm 6a789b89-c2b8-45f5-82cc-fb80234d0c6c 2026-03-20T11:46:16.790 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.792+0000 7f339633c900 20 rados->read ofs=0 len=0 2026-03-20T11:46:16.790 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.792+0000 7f339633c900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:46:16.790 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.792+0000 7f339633c900 20 rados->read ofs=0 len=0 2026-03-20T11:46:16.791 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.793+0000 7f339633c900 20 rados_obj.operate() r=0 bl.length=10 2026-03-20T11:46:16.791 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.793+0000 7f339633c900 20 rados->read ofs=0 len=0 2026-03-20T11:46:16.791 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.793+0000 7f339633c900 20 rados_obj.operate() r=0 bl.length=2020 2026-03-20T11:46:16.791 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.793+0000 7f339633c900 20 rados->read ofs=0 len=0 2026-03-20T11:46:16.791 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.793+0000 7f339633c900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:46:16.791 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.793+0000 7f339633c900 20 rados->read ofs=0 len=0 2026-03-20T11:46:16.792 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.794+0000 7f339633c900 20 rados_obj.operate() r=0 bl.length=1030 2026-03-20T11:46:16.792 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.794+0000 7f339633c900 20 zone b2 found 2026-03-20T11:46:16.792 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.794+0000 7f339633c900 4 Realm: test-realm (6a789b89-c2b8-45f5-82cc-fb80234d0c6c) 2026-03-20T11:46:16.792 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.794+0000 7f339633c900 4 ZoneGroup: b (b) 2026-03-20T11:46:16.792 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.794+0000 7f339633c900 4 Zone: b2 (cd52b229-0d4a-43bf-b4ba-156aa19e7aef) 2026-03-20T11:46:16.792 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.794+0000 7f339633c900 4 using period configuration: cb2fcdce-2c1b-429a-8601-a284ee3a2381:12 2026-03-20T11:46:16.792 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.794+0000 7f339633c900 20 period zonegroup name b 2026-03-20T11:46:16.792 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.794+0000 7f339633c900 20 using current period zonegroup b 2026-03-20T11:46:16.792 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.794+0000 7f339633c900 20 rados->read ofs=0 len=0 2026-03-20T11:46:16.793 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.794+0000 7f339633c900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:16.793 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.794+0000 7f339633c900 20 rados->read ofs=0 len=0 2026-03-20T11:46:16.793 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.795+0000 7f339633c900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:16.793 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.795+0000 7f339633c900 20 init(): pipe_map (zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:46:16.793 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.795+0000 7f339633c900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:16.793 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.795+0000 7f339633c900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:16.793 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.795+0000 7f339633c900 20 rados->read ofs=0 len=0 2026-03-20T11:46:16.793 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.795+0000 7f339633c900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:16.793 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.795+0000 7f339633c900 20 rados->read ofs=0 len=0 2026-03-20T11:46:16.793 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.795+0000 7f339633c900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:16.793 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.795+0000 7f339633c900 20 init(): pipe_map (zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:46:16.793 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.795+0000 7f339633c900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:16.793 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.795+0000 7f339633c900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:16.794 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.795+0000 7f339633c900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:16.794 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.795+0000 7f339633c900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:16.794 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.795+0000 7f339633c900 20 started sync module instance, tier type = 2026-03-20T11:46:16.794 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.795+0000 7f339633c900 20 generating connection object for zone b1 id efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c 2026-03-20T11:46:16.794 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.795+0000 7f339633c900 20 started zone id=cd52b229-0d4a-43bf-b4ba-156aa19e7aef (name=b2) with tier type = 2026-03-20T11:46:16.797 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.799+0000 7f339633c900 20 add_watcher() i=4 2026-03-20T11:46:16.797 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.799+0000 7f339633c900 20 add_watcher() i=3 2026-03-20T11:46:16.797 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.799+0000 7f339633c900 20 add_watcher() i=1 2026-03-20T11:46:16.797 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.799+0000 7f339633c900 20 add_watcher() i=5 2026-03-20T11:46:16.797 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.800+0000 7f339633c900 20 add_watcher() i=2 2026-03-20T11:46:16.798 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.800+0000 7f339633c900 20 add_watcher() i=0 2026-03-20T11:46:16.798 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.800+0000 7f339633c900 20 add_watcher() i=7 2026-03-20T11:46:16.798 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.800+0000 7f339633c900 20 add_watcher() i=6 2026-03-20T11:46:16.798 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.800+0000 7f339633c900 2 all 8 watchers are set, enabling cache 2026-03-20T11:46:16.801 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.803+0000 7f339633c900 20 rgw_check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2026-03-20T11:46:16.801 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.803+0000 7f339633c900 20 rgw_check_secure_mon_conn(): mode 1 is insecure 2026-03-20T11:46:16.801 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.803+0000 7f339633c900 5 note: GC not initialized 2026-03-20T11:46:16.843 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.844+0000 7f339633c900 20 init_complete bucket index max shards: 11 2026-03-20T11:46:16.843 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.844+0000 7f339633c900 20 Filter name: none 2026-03-20T11:46:16.843 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.844+0000 7f32f37fe640 20 reqs_thread_entry: start 2026-03-20T11:46:16.843 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.845+0000 7f32f1ffb640 20 reqs_thread_entry: start 2026-03-20T11:46:16.844 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.845+0000 7f339633c900 10 rgw bucket trim manager: Watching bilog.trim 2026-03-20T11:46:16.844 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.845+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3cccd7d0:12BucketTrimCR: operate() 2026-03-20T11:46:16.844 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.845+0000 7f339633c900 10 fetching active bucket counters 2026-03-20T11:46:16.844 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.845+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd0280:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:16.844 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.845+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd0280:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:16.844 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.846+0000 7f339633c900 20 run: stack=0x556b3cccded0 is io blocked 2026-03-20T11:46:16.844 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.846+0000 7f336d7fa640 20 trim: get_bucket_counters: [] 2026-03-20T11:46:16.844 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.846+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd0280:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:16.844 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.846+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd0280:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:16.844 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.846+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd0280:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:16.844 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.846+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd0280:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:16.844 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.846+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3cccd7d0:12BucketTrimCR: operate() 2026-03-20T11:46:16.844 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.846+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:16.844 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.846+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:16.844 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.846+0000 7f339633c900 20 run: stack=0x556b3cccded0 is io blocked 2026-03-20T11:46:16.845 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.847+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:16.845 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.847+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:16.845 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.847+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:16.845 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.847+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:16.845 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.847+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3cccd7d0:12BucketTrimCR: operate() 2026-03-20T11:46:16.845 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.847+0000 7f339633c900 10 listing cold buckets from marker= 2026-03-20T11:46:16.845 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.847+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:14MetadataListCR: operate() 2026-03-20T11:46:16.845 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.847+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:14MetadataListCR: operate() 2026-03-20T11:46:16.845 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.847+0000 7f339633c900 20 enqueued request req=0x7f3328001570 2026-03-20T11:46:16.845 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.847+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:16.845 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.847+0000 7f339633c900 20 req: 0x7f3328001570 2026-03-20T11:46:16.845 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.847+0000 7f339633c900 20 run: stack=0x556b3cccded0 is io blocked 2026-03-20T11:46:16.845 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.847+0000 7f3341fe3640 20 dequeued request req=0x7f3328001570 2026-03-20T11:46:16.845 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.847+0000 7f3341fe3640 20 RGWWQ: empty 2026-03-20T11:46:16.845 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.847+0000 7f3341fe3640 20 rgw async rados processor: starting metadata listing at 2026-03-20T11:46:16.852 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.854+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-10 2026-03-20T11:46:16.852 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.854+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:16.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.860+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:16.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.866+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:16.870 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.872+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:16.877 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.879+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:16.884 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.886+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:16.890 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.892+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-19 2026-03-20T11:46:16.890 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.892+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-3 2026-03-20T11:46:16.890 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.892+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:16.895 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.897+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:16.900 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.902+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-17 2026-03-20T11:46:16.900 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.902+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:16.905 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.907+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-38 2026-03-20T11:46:16.905 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.907+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:16.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.912+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-24 2026-03-20T11:46:16.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.912+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-36 2026-03-20T11:46:16.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.912+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:16.915 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.917+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:16.921 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.923+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:16.926 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.928+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-44 2026-03-20T11:46:16.926 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.928+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-35 2026-03-20T11:46:16.926 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.928+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:16.931 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.933+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-16 2026-03-20T11:46:16.931 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.933+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:16.935 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.938+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:16.940 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.942+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 2026-03-20T11:46:16.945 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.947+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:16.949 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.951+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-55 2026-03-20T11:46:16.949 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.951+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-13 2026-03-20T11:46:16.950 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.951+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-48 2026-03-20T11:46:16.950 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.951+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-25 2026-03-20T11:46:16.950 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.951+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-51 2026-03-20T11:46:16.950 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.951+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:16.954 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.956+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:16.958 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.960+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-46 2026-03-20T11:46:16.958 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.960+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:46:16.962 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.964+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:46:16.966 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.968+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:46:16.970 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.972+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-26 2026-03-20T11:46:16.970 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.972+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-20 2026-03-20T11:46:16.971 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.972+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:46:16.975 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.977+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:46:16.979 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.981+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-1 2026-03-20T11:46:16.979 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.981+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:46:16.983 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.985+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:46:16.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.992+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:46:16.995 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.997+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-52 2026-03-20T11:46:16.995 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.997+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-32 2026-03-20T11:46:16.995 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.997+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-42 2026-03-20T11:46:16.995 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:16.997+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:46:17.000 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.002+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:46:17.004 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.006+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-7 2026-03-20T11:46:17.004 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.006+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-54 2026-03-20T11:46:17.004 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.006+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-49 2026-03-20T11:46:17.004 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.006+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-47 2026-03-20T11:46:17.004 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.006+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 2026-03-20T11:46:17.008 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.010+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:46:17.011 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.013+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-11 2026-03-20T11:46:17.011 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.013+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-12 2026-03-20T11:46:17.011 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.013+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-41 2026-03-20T11:46:17.011 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.013+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-23 2026-03-20T11:46:17.011 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.013+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-15 2026-03-20T11:46:17.011 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.013+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-45 2026-03-20T11:46:17.011 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.013+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:46:17.015 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.017+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:46:17.018 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.020+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-9 2026-03-20T11:46:17.018 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.020+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:46:17.021 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.023+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-29 2026-03-20T11:46:17.021 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.023+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:46:17.024 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.026+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-31 2026-03-20T11:46:17.025 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.026+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:46:17.027 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.029+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-33 2026-03-20T11:46:17.027 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.029+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:17.029 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.031+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-4 2026-03-20T11:46:17.030 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.031+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:46:17.032 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.034+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:17.034 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.036+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:17.037 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.039+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-43 2026-03-20T11:46:17.037 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.039+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-30 2026-03-20T11:46:17.037 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.039+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-18 2026-03-20T11:46:17.037 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.039+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-2 2026-03-20T11:46:17.037 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.039+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:46:17.039 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.041+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-8 2026-03-20T11:46:17.039 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.041+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:46:17.041 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.043+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:17.043 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.045+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-58 2026-03-20T11:46:17.043 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.045+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:46:17.044 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.046+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 2026-03-20T11:46:17.046 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.048+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-27 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-28 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-59 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f3341fe3640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:14MetadataListCR: operate() 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:14MetadataListCR: operate() 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:14MetadataListCR: operate() 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:14MetadataListCR: operate() 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3cccd7d0:12BucketTrimCR: operate() 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 4 collected 50 buckets for trim 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 4 starting trim on bucket=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 4 starting trim on bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 cr:s=0x556b3cbe9a60:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 4 starting trim on bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 4 starting trim on bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccd1bc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd66e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 cr:s=0x556b3cbe9a60:op=0x556b3ccd7100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd7b20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccd1bc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 enqueued request req=0x556b3ccd84d0 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 req: 0x556b3ccd84d0 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is io blocked 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd66e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 enqueued request req=0x556b3ccd8870 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 req: 0x556b3ccd84d0 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 req: 0x556b3ccd8870 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 run: stack=0x556b3cd15360 is io blocked 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 cr:s=0x556b3cbe9a60:op=0x556b3ccd7100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 enqueued request req=0x556b3ccd8c10 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 req: 0x556b3ccd84d0 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 req: 0x556b3ccd8870 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 req: 0x556b3ccd8c10 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 run: stack=0x556b3cbe9a60 is io blocked 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd7b20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.048 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 enqueued request req=0x556b3c887640 2026-03-20T11:46:17.049 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.049 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 req: 0x556b3ccd84d0 2026-03-20T11:46:17.049 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 req: 0x556b3ccd8870 2026-03-20T11:46:17.049 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 req: 0x556b3ccd8c10 2026-03-20T11:46:17.049 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 req: 0x556b3c887640 2026-03-20T11:46:17.049 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is io blocked 2026-03-20T11:46:17.049 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f33597fa640 20 dequeued request req=0x556b3ccd84d0 2026-03-20T11:46:17.049 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f33597fa640 20 RGWWQ: 2026-03-20T11:46:17.049 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f33597fa640 20 req: 0x556b3ccd8870 2026-03-20T11:46:17.049 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f33597fa640 20 req: 0x556b3ccd8c10 2026-03-20T11:46:17.049 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f33597fa640 20 req: 0x556b3c887640 2026-03-20T11:46:17.049 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f3358ff9640 20 dequeued request req=0x556b3ccd8870 2026-03-20T11:46:17.049 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f3358ff9640 20 RGWWQ: 2026-03-20T11:46:17.049 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f3358ff9640 20 req: 0x556b3ccd8c10 2026-03-20T11:46:17.049 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f3358ff9640 20 req: 0x556b3c887640 2026-03-20T11:46:17.049 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f334ffff640 20 dequeued request req=0x556b3ccd8c10 2026-03-20T11:46:17.049 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f334ffff640 20 RGWWQ: 2026-03-20T11:46:17.049 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f334ffff640 20 req: 0x556b3c887640 2026-03-20T11:46:17.049 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f334f7fe640 20 dequeued request req=0x556b3c887640 2026-03-20T11:46:17.049 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f334f7fe640 20 RGWWQ: empty 2026-03-20T11:46:17.049 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f334ffff640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 : miss 2026-03-20T11:46:17.049 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.049+0000 7f3358ff9640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 : miss 2026-03-20T11:46:17.049 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f334f7fe640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 : miss 2026-03-20T11:46:17.049 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f33597fa640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 : miss 2026-03-20T11:46:17.049 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f334f7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.049 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f334ffff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.049 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f33597fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.049 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f3358ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f334f7fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f334f7fe640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 info.flags=0x17 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f334f7fe640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 to cache LRU end 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f334f7fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f334f7fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f334f7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f334f7fe640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-55 : miss 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f334f7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f33597fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=387 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f33597fa640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 info.flags=0x17 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f33597fa640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 to cache LRU end 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f33597fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f33597fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f33597fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f33597fa640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-60 : miss 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f33597fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f334ffff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f334ffff640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 info.flags=0x17 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f334ffff640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 to cache LRU end 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f334ffff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f334ffff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f334ffff640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f334ffff640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-54 : miss 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f334ffff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.050+0000 7f3358ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f3358ff9640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 info.flags=0x17 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f3358ff9640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 to cache LRU end 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f3358ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f3358ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f3358ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f3358ff9640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-53 : miss 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f3358ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f3358ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f3358ff9640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-53 info.flags=0x0 2026-03-20T11:46:17.051 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f3358ff9640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-53 to cache LRU end 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f3358ff9640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-53 : miss 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f3358ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f334f7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f334f7fe640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-55 info.flags=0x0 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f334f7fe640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-55 to cache LRU end 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f334f7fe640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-55 : miss 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f334f7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f334f7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f334f7fe640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-55 info.flags=0x0 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f334f7fe640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-55 to cache LRU end 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f334f7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding source pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f334f7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding dest pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f334f7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f334f7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f334f7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd7b20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd7b20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd7b20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd7b20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f339633c900 20 cr:s=0x556b3ccd2820:op=0x556b3c887a60:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f339633c900 20 cr:s=0x556b3ccd2820:op=0x556b3ccd7b20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f339633c900 20 cr:s=0x556b3ccd2820:op=0x556b3ccd7b20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f3358ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f3358ff9640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-53 info.flags=0x0 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f3358ff9640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-53 to cache LRU end 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f3358ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13): adding source pipe: {s={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f3358ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13): adding dest pipe: {s={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = b8c0214d9c98936a8141b297c3bef91e61c9f57ec02595335b43e23b6b6b9217 2026-03-20T11:46:17.052 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:b8c0214d9c98936a8141b297c3bef91e61c9f57ec02595335b43e23b6b6b9217 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.051+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 8d6fa2f57903a0d62f11817997d090e295cc5c589ee63e315e4207b45742b7ac 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=8d6fa2f57903a0d62f11817997d090e295cc5c589ee63e315e4207b45742b7ac 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f3358ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f3358ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=0, curl_handle=0x556b3c889a20 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 run: stack=0x556b3ccd2820 is io blocked 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f32f1ffb640 20 link_request req_data=0x556b3c88a5f0 req_data->id=0, curl_handle=0x556b3c889a20 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f3358ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd66e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd66e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd66e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd66e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 cr:s=0x556b3c88b1d0:op=0x556b3c88abb0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 run: stack=0x556b3cd15360 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 cr:s=0x556b3c88b1d0:op=0x556b3c88b360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 cr:s=0x556b3c88b1d0:op=0x556b3c88b360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.053 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 0b14bd97522b310c56cc53ce33b616ff99c1ac580e129129484cf418a826b600 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:0b14bd97522b310c56cc53ce33b616ff99c1ac580e129129484cf418a826b600 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 46a751c1f4482fa0445389daf8ad2c0d5e1c3e237e456e518637786689d1c230 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=46a751c1f4482fa0445389daf8ad2c0d5e1c3e237e456e518637786689d1c230 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=1, curl_handle=0x556b3cce44b0 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 run: stack=0x556b3c88b1d0 is io blocked 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f33597fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f33597fa640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-60 info.flags=0x0 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f33597fa640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-60 to cache LRU end 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f33597fa640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-60 : miss 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f33597fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f334ffff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f334ffff640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-54 info.flags=0x0 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f334ffff640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-54 to cache LRU end 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f334ffff640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-54 : miss 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f334ffff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f32f1ffb640 20 link_request req_data=0x556b3cce4bc0 req_data->id=1, curl_handle=0x556b3cce44b0 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f33597fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f33597fa640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-60 info.flags=0x0 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f33597fa640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-60 to cache LRU end 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f33597fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20): adding source pipe: {s={b=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f33597fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20): adding dest pipe: {s={b=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f33597fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f33597fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f33597fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccd1bc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccd1bc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccd1bc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccd1bc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 cr:s=0x556b3ccd2070:op=0x556b3cce6260:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 cr:s=0x556b3ccd2070:op=0x556b3cce6880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 cr:s=0x556b3ccd2070:op=0x556b3cce6880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.054 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = cf2d4a7672a7ccd87689810907553b991418d5cc0c4844e5e71801166a294b09 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:cf2d4a7672a7ccd87689810907553b991418d5cc0c4844e5e71801166a294b09 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 73591568410779fe8871295e59c8ca55c34dc24555091c7549a95acfe23ce068 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=73591568410779fe8871295e59c8ca55c34dc24555091c7549a95acfe23ce068 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-60%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=2, curl_handle=0x556b3cce8410 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f339633c900 20 run: stack=0x556b3ccd2070 is io blocked 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.052+0000 7f32f1ffb640 20 link_request req_data=0x556b3cce8b20 req_data->id=2, curl_handle=0x556b3cce8410 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f334ffff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f334ffff640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-54 info.flags=0x0 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f334ffff640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-54 to cache LRU end 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f334ffff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14): adding source pipe: {s={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.055 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f334ffff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14): adding dest pipe: {s={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.056 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f334ffff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.056 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f334ffff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.056 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f334ffff640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:17.056 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3cbe9a60:op=0x556b3ccd7100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.056 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3cbe9a60:op=0x556b3ccd7100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.056 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3cbe9a60:op=0x556b3ccd7100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.056 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3cbe9a60:op=0x556b3ccd7100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.056 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3cbe9a60:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.056 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3ccd76f0:op=0x556b3ccea1c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.056 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3cbe9a60:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.056 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 run: stack=0x556b3cbe9a60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.056 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3ccd76f0:op=0x556b3ccea810:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.056 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3ccd76f0:op=0x556b3ccea810:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.056 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.056 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.056 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.056 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.056 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.056 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 2a1e6cd71835e16ad1d9364024ff374ce201af9ad25c414c94f3b155be136506 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2a1e6cd71835e16ad1d9364024ff374ce201af9ad25c414c94f3b155be136506 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 5a2023afb7e6a8a89790b8368ed65c3ca28a067aa55413226788c255d68b2d9d 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5a2023afb7e6a8a89790b8368ed65c3ca28a067aa55413226788c255d68b2d9d 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=3, curl_handle=0x556b3cce8410 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 run: stack=0x556b3ccd76f0 is io blocked 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3ccd2070:op=0x556b3cce6880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3ccd2070:op=0x556b3cce6880:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3ccd2070:op=0x556b3cce6260:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 15 stack 0x556b3ccd2070 end 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 run: stack=0x556b3ccd2070 is done 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 collect(): s=0x556b3cbe9e50 stack=0x556b3ccd2070 is complete 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f32f1ffb640 20 link_request req_data=0x556b3ccecae0 req_data->id=3, curl_handle=0x556b3cce8410 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce6260:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.0 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.1 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.2 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.3 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.4 2026-03-20T11:46:17.057 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.5 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.6 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.7 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.8 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.9 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20.10 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3ccd2070:op=0x556b3cce6850:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cce6e40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3cced070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3ccee530:op=0x556b3ccedb40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3ccef230:op=0x556b3ccee840:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3cceff50:op=0x556b3ccef560:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3ccf0c70:op=0x556b3ccf0280:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3ccf1990:op=0x556b3ccf0fa0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3ccf1cc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3ccf33b0:op=0x556b3ccf29c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3cd1c9c0:op=0x556b3cd1bfd0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3ccd2070:op=0x556b3cce6850:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 run: stack=0x556b3ccd2070 is io blocked 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cce6e40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 run: stack=0x556b3cce7430 is io blocked 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3cced070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 run: stack=0x556b3cced850 is io blocked 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3ccee530:op=0x556b3ccedb40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 run: stack=0x556b3ccee530 is io blocked 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3ccef230:op=0x556b3ccee840:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 run: stack=0x556b3ccef230 is io blocked 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3cceff50:op=0x556b3ccef560:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 run: stack=0x556b3cceff50 is io blocked 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3ccf0c70:op=0x556b3ccf0280:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 run: stack=0x556b3ccf0c70 is io blocked 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3ccf1990:op=0x556b3ccf0fa0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 run: stack=0x556b3ccf1990 is io blocked 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3ccf1cc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 run: stack=0x556b3ccf26b0 is io blocked 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3ccf33b0:op=0x556b3ccf29c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 run: stack=0x556b3ccf33b0 is io blocked 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.053+0000 7f339633c900 20 cr:s=0x556b3cd1c9c0:op=0x556b3cd1bfd0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.058 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.054+0000 7f339633c900 20 run: stack=0x556b3cd1c9c0 is io blocked 2026-03-20T11:46:17.058 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.057+0000 7f76ffcf1640 -1 req 3070508590505100628 0.004000004s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.057+0000 7f339633c900 20 cr:s=0x556b3cceff50:op=0x556b3ccef560:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3cceff50:op=0x556b3ccef560:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3ccf1cc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccee530:op=0x556b3ccedb40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccef230:op=0x556b3ccee840:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccf1990:op=0x556b3ccf0fa0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccd2820:op=0x556b3ccd7b20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccd2820:op=0x556b3ccd7b20:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3c88b1d0:op=0x556b3c88b360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3cceff50:op=0x556b3ccef560:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3ccf1cc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccee530:op=0x556b3ccedb40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccef230:op=0x556b3ccee840:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccf1990:op=0x556b3ccf0fa0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccd2820:op=0x556b3c887a60:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 15 stack 0x556b3ccd2820 end 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 run: stack=0x556b3ccd2820 is done 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3c88b1d0:op=0x556b3c88b360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3cceff50:op=0x556b3ccef560:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 15 stack 0x556b3cceff50 end 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 run: stack=0x556b3cceff50 is done 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3ccf1cc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccee530:op=0x556b3ccedb40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccef230:op=0x556b3ccee840:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccf1990:op=0x556b3ccf0fa0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 collect(): s=0x556b3ccd07e0 stack=0x556b3ccd2820 is complete 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 15 stack 0x556b3ccd07e0 end 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is done 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3c88b1d0:op=0x556b3c88b360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce6260:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3ccf1cc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 15 stack 0x556b3ccf26b0 end 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 run: stack=0x556b3ccf26b0 is done 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccee530:op=0x556b3ccedb40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 15 stack 0x556b3ccee530 end 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 run: stack=0x556b3ccee530 is done 2026-03-20T11:46:17.060 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccef230:op=0x556b3ccee840:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 15 stack 0x556b3ccef230 end 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 run: stack=0x556b3ccef230 is done 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccf1990:op=0x556b3ccf0fa0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 15 stack 0x556b3ccf1990 end 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 run: stack=0x556b3ccf1990 is done 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3c88b1d0:op=0x556b3c88b360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce6260:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 4 starting trim on bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3c88b1d0:op=0x556b3c88abb0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 15 stack 0x556b3c88b1d0 end 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 run: stack=0x556b3c88b1d0 is done 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce6260:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 collect(): s=0x556b3cd15360 stack=0x556b3c88b1d0 is complete 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 10 trimming bilogs for bucket=:dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) markers=[00000000002.32.1,00000000002.59.1,00000000002.36.1,00000000002.31.1,00000000002.37.1,00000000002.34.1,00000000002.61.1,00000000002.37.1,00000000002.34.1,00000000002.53.1,00000000002.52.1], shards=11 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce6260:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 enqueued request req=0x556b3ccee130 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 req: 0x556b3ccee130 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is io blocked 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd7ab0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 10 trimming bilog shard 0 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.32.1 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 10 trimming bilog shard 1 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.59.1 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 10 trimming bilog shard 2 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.36.1 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 10 trimming bilog shard 3 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.31.1 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 10 trimming bilog shard 4 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.37.1 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 10 trimming bilog shard 5 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.34.1 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 10 trimming bilog shard 6 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.61.1 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 10 trimming bilog shard 7 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.37.1 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 10 trimming bilog shard 8 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.34.1 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 10 trimming bilog shard 9 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.53.1 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 10 trimming bilog shard 10 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.52.1 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 run: stack=0x556b3cd15360 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce6260:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccf1990:op=0x556b3cd25d90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3c88b1d0:op=0x556b3cd1e5c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccef230:op=0x556b3ccf0f60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccee530:op=0x556b3cd25070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3cceff50:op=0x556b3cd31330:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccd2820:op=0x556b3cd321d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3c887a60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3cd20e20:op=0x556b3cd2daa0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3cce7be0:op=0x556b3cd2e2a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccec200:op=0x556b3cd37790:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3cd2eaa0:op=0x556b3cd37f90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccf1990:op=0x556b3cd25d90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.061 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.0 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f334effd640 20 dequeued request req=0x556b3ccee130 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f334effd640 20 RGWWQ: empty 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 run: stack=0x556b3ccf1990 is io blocked 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3c88b1d0:op=0x556b3cd1e5c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.1 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 run: stack=0x556b3c88b1d0 is io blocked 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccef230:op=0x556b3ccf0f60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.2 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 run: stack=0x556b3ccef230 is io blocked 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccee530:op=0x556b3cd25070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.3 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 run: stack=0x556b3ccee530 is io blocked 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3cceff50:op=0x556b3cd31330:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.4 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 run: stack=0x556b3cceff50 is io blocked 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccd2820:op=0x556b3cd321d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.5 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 run: stack=0x556b3ccd2820 is io blocked 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3c887a60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.6 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is io blocked 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3cd20e20:op=0x556b3cd2daa0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.7 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 run: stack=0x556b3cd20e20 is io blocked 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3cce7be0:op=0x556b3cd2e2a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.8 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 run: stack=0x556b3cce7be0 is io blocked 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccec200:op=0x556b3cd37790:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.9 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 run: stack=0x556b3ccec200 is io blocked 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3cd2eaa0:op=0x556b3cd37f90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.10 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 run: stack=0x556b3cd2eaa0 is io blocked 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccd2070:op=0x556b3cce6850:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccd2070:op=0x556b3cce6850:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccd2070:op=0x556b3cce6850:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3ccd2070:op=0x556b3cce6850:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 15 stack 0x556b3ccd2070 end 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 run: stack=0x556b3ccd2070 is done 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce6260:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.058+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.059+0000 7f334effd640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 : miss 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.059+0000 7f334effd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cd1c9c0:op=0x556b3cd1bfd0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.062 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cd1c9c0:op=0x556b3cd1bfd0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cd1c9c0:op=0x556b3cd1bfd0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cd1c9c0:op=0x556b3cd1bfd0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 15 stack 0x556b3cd1c9c0 end 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 run: stack=0x556b3cd1c9c0 is done 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce6260:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3c88b1d0:op=0x556b3cd1e5c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3c88b1d0:op=0x556b3cd1e5c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3c887a60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3c88b1d0:op=0x556b3cd1e5c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3c887a60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3c88b1d0:op=0x556b3cd1e5c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 15 stack 0x556b3c88b1d0 end 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 run: stack=0x556b3c88b1d0 is done 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3c887a60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd7ab0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 run: stack=0x556b3cd15360 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3c887a60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 15 stack 0x556b3cd2fd80 end 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is done 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd7ab0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 run: stack=0x556b3cd15360 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3cced070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3cced070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cce6e40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3ccf33b0:op=0x556b3ccf29c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3ccf0c70:op=0x556b3ccf0280:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3ccf1990:op=0x556b3cd25d90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3ccef230:op=0x556b3ccf0f60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3ccd2820:op=0x556b3cd321d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cd20e20:op=0x556b3cd2daa0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cd2eaa0:op=0x556b3cd37f90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3ccec200:op=0x556b3cd37790:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3cced070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cce6e40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3ccf33b0:op=0x556b3ccf29c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3ccf0c70:op=0x556b3ccf0280:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3ccf1990:op=0x556b3cd25d90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3ccef230:op=0x556b3ccf0f60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3ccd2820:op=0x556b3cd321d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cd20e20:op=0x556b3cd2daa0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cd2eaa0:op=0x556b3cd37f90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3ccec200:op=0x556b3cd37790:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3cced070:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 15 stack 0x556b3cced850 end 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 run: stack=0x556b3cced850 is done 2026-03-20T11:46:17.063 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cce6e40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3ccf33b0:op=0x556b3ccf29c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3ccf0c70:op=0x556b3ccf0280:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3ccf1990:op=0x556b3cd25d90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3ccef230:op=0x556b3ccf0f60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3ccd2820:op=0x556b3cd321d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cd20e20:op=0x556b3cd2daa0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cd2eaa0:op=0x556b3cd37f90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3ccec200:op=0x556b3cd37790:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce6260:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cce6e40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 15 stack 0x556b3cce7430 end 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 run: stack=0x556b3cce7430 is done 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3ccf33b0:op=0x556b3ccf29c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 15 stack 0x556b3ccf33b0 end 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 run: stack=0x556b3ccf33b0 is done 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3ccf0c70:op=0x556b3ccf0280:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 15 stack 0x556b3ccf0c70 end 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 run: stack=0x556b3ccf0c70 is done 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3ccf1990:op=0x556b3cd25d90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 15 stack 0x556b3ccf1990 end 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 run: stack=0x556b3ccf1990 is done 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3ccef230:op=0x556b3ccf0f60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 15 stack 0x556b3ccef230 end 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 run: stack=0x556b3ccef230 is done 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3ccd2820:op=0x556b3cd321d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 15 stack 0x556b3ccd2820 end 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 run: stack=0x556b3ccd2820 is done 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cd20e20:op=0x556b3cd2daa0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 15 stack 0x556b3cd20e20 end 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 run: stack=0x556b3cd20e20 is done 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cd2eaa0:op=0x556b3cd37f90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 15 stack 0x556b3cd2eaa0 end 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 run: stack=0x556b3cd2eaa0 is done 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3ccec200:op=0x556b3cd37790:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 15 stack 0x556b3ccec200 end 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 run: stack=0x556b3ccec200 is done 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce6260:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:17.064 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd7ab0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:17.065 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce6260:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:17.065 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd7ab0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:17.065 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce6260:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:17.065 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd7ab0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:17.065 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.065 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd7ab0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:17.065 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccef520:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:17.065 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd7ab0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:17.065 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccef520:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:17.065 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 enqueued request req=0x556b3ccd2340 2026-03-20T11:46:17.065 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.065 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 req: 0x556b3ccd2340 2026-03-20T11:46:17.065 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is io blocked 2026-03-20T11:46:17.065 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd7ab0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:17.065 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f339633c900 20 run: stack=0x556b3cd15360 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.065 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f334effd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.065 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f334effd640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 info.flags=0x17 2026-03-20T11:46:17.065 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f334effd640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 to cache LRU end 2026-03-20T11:46:17.065 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f334effd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.065 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f334effd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:17.065 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f334effd640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:17.065 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f334effd640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-28 : miss 2026-03-20T11:46:17.065 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f334e7fc640 20 dequeued request req=0x556b3ccd2340 2026-03-20T11:46:17.065 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f334e7fc640 20 RGWWQ: empty 2026-03-20T11:46:17.065 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.061+0000 7f334effd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.067 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.062+0000 7f334effd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.067 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.062+0000 7f334effd640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-28 info.flags=0x0 2026-03-20T11:46:17.067 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.062+0000 7f334effd640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-28 to cache LRU end 2026-03-20T11:46:17.067 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.062+0000 7f334effd640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-28 : miss 2026-03-20T11:46:17.067 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.062+0000 7f334effd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.067 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.062+0000 7f339633c900 20 cr:s=0x556b3ccee530:op=0x556b3cd25070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.067 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.062+0000 7f339633c900 20 cr:s=0x556b3ccee530:op=0x556b3cd25070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.067 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.062+0000 7f339633c900 20 cr:s=0x556b3ccee530:op=0x556b3cd25070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.067 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.062+0000 7f339633c900 20 cr:s=0x556b3ccee530:op=0x556b3cd25070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.067 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.062+0000 7f339633c900 15 stack 0x556b3ccee530 end 2026-03-20T11:46:17.067 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.062+0000 7f339633c900 20 run: stack=0x556b3ccee530 is done 2026-03-20T11:46:17.067 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.062+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd7ab0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:17.067 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.062+0000 7f339633c900 20 run: stack=0x556b3cd15360 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.067 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.062+0000 7f334e7fc640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 info.flags=0x11 2026-03-20T11:46:17.067 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.062+0000 7f334e7fc640 10 rgw async rados processor: moving b2.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 to cache LRU end 2026-03-20T11:46:17.067 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.062+0000 7f334e7fc640 10 rgw async rados processor: distributing notification oid=b2.rgw.control:notify.6 cni=[op: 0, obj: b2.rgw.meta:root:.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20, ofs0, ns] 2026-03-20T11:46:17.067 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3cceff50:op=0x556b3cd31330:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.067 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f334effd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.067 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f334effd640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-28 info.flags=0x0 2026-03-20T11:46:17.067 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3cceff50:op=0x556b3cd31330:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.067 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3cceff50:op=0x556b3cd31330:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.067 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3cceff50:op=0x556b3cd31330:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.067 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 15 stack 0x556b3cceff50 end 2026-03-20T11:46:17.067 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f334effd640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-28 to cache LRU end 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 run: stack=0x556b3cceff50 is done 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd7ab0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 run: stack=0x556b3cd15360 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f334effd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding source pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f334effd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding dest pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f334effd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f334effd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f334effd640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3ccf2980:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 491e80ef69d7dc06b61a218ae32d1fddd2b578b88a9350f8f64174f03743ea0f 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.068 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:491e80ef69d7dc06b61a218ae32d1fddd2b578b88a9350f8f64174f03743ea0f 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = b4bd0d36a02150b39c6c587ea02cf07d86a2a2c188a316ac9c33b4d033432b59 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b4bd0d36a02150b39c6c587ea02cf07d86a2a2c188a316ac9c33b4d033432b59 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=4, curl_handle=0x556b3cce44b0 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 run: stack=0x556b3cce7430 is io blocked 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f336dffb640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 257698037999 cookie 93919070221568 notifier 4705 bl.length()=567 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f336dffb640 10 rgw watcher librados: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 info.flags=0x11 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f336dffb640 10 rgw watcher librados: moving b2.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 to cache LRU end 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f32f1ffb640 20 link_request req_data=0x556b3cce4bc0 req_data->id=4, curl_handle=0x556b3cce44b0 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3cce7be0:op=0x556b3cd2e2a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3cce7be0:op=0x556b3cd2e2a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3cce7be0:op=0x556b3cd2e2a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3cce7be0:op=0x556b3cd2e2a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 15 stack 0x556b3cce7be0 end 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 run: stack=0x556b3cce7be0 is done 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd7ab0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 trim: trimmed bucket instance dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 15 stack 0x556b3cd15360 end 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 run: stack=0x556b3cd15360 is done 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 4 starting trim on bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3cd30570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3cd30570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 enqueued request req=0x556b3cced660 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 req: 0x556b3cced660 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 run: stack=0x556b3cd15360 is io blocked 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f334e7fc640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-60[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20]): orig_sources= new_sources= 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f334e7fc640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-60[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20]): potential sources added=[] removed=[] 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f334e7fc640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-60[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20]): orig_dests= new_dests= 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f334e7fc640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-60[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20]): potential dests added=[] removed=[] 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f334e7fc640 20 dequeued request req=0x556b3cced660 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f334e7fc640 20 RGWWQ: empty 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccef520:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:17.069 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccef520:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccef520:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccef520:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd20340:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f334e7fc640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 : miss 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f334e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd20340:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 enqueued request req=0x556b3c813f40 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 req: 0x556b3c813f40 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is io blocked 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f334d7fa640 20 dequeued request req=0x556b3c813f40 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.063+0000 7f334d7fa640 20 RGWWQ: empty 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.064+0000 7f334e7fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.064+0000 7f334e7fc640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 info.flags=0x17 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.064+0000 7f334e7fc640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 to cache LRU end 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.064+0000 7f334e7fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.064+0000 7f334e7fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.064+0000 7f334e7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.064+0000 7f334e7fc640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-31 : miss 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.064+0000 7f334e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.064+0000 7f334e7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.064+0000 7f334e7fc640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-31 info.flags=0x0 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.064+0000 7f334e7fc640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-31 to cache LRU end 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.064+0000 7f334e7fc640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-31 : miss 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.064+0000 7f334e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f334d7fa640 10 rgw async rados processor: removing b2.rgw.meta+root+.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 from cache 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f334d7fa640 10 rgw async rados processor: distributing notification oid=b2.rgw.control:notify.6 cni=[op: 1, obj: b2.rgw.meta:root:.bucket.meta.dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20, ofs0, ns] 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f334e7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f334e7fc640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-31 info.flags=0x0 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f334e7fc640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-31 to cache LRU end 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f334e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding source pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f334e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding dest pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f334e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f334e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f334e7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3cd30570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3cd30570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3cd30570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3cd30570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.070 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3c88ab90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 20 run: stack=0x556b3cd15360 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3cd240e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3cd240e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 06cf23774ab25af8380c9298dc1a368621558f66be1e63c5980ef80592b42058 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:06cf23774ab25af8380c9298dc1a368621558f66be1e63c5980ef80592b42058 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 2ef58e87233479f4613b7211d1639899aba03e66ba44c7bed2732b6835708eeb 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2ef58e87233479f4613b7211d1639899aba03e66ba44c7bed2732b6835708eeb 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=5, curl_handle=0x556b3c889a20 2026-03-20T11:46:17.071 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f339633c900 20 run: stack=0x556b3cced850 is io blocked 2026-03-20T11:46:17.072 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.065+0000 7f76c7480640 -1 req 9774451212059919700 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.072 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f32f1ffb640 20 link_request req_data=0x556b3c88a5f0 req_data->id=5, curl_handle=0x556b3c889a20 2026-03-20T11:46:17.072 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.065+0000 7f336dffb640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 257698038000 cookie 93919070221568 notifier 4705 bl.length()=206 2026-03-20T11:46:17.072 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.066+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd20340:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:17.072 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.066+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd20340:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:17.072 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.066+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd20340:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:17.072 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.066+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd20340:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:17.072 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.066+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.072 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.066+0000 7f339633c900 20 trim: trimmed bucket instance dipepu-60:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.20 2026-03-20T11:46:17.072 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.066+0000 7f339633c900 15 stack 0x556b3cbe9e50 end 2026-03-20T11:46:17.072 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.066+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is done 2026-03-20T11:46:17.072 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.066+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.072 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.066+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.072 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.066+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.072 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.066+0000 7f339633c900 4 starting trim on bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:17.072 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.066+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd30570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.072 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.066+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd30570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.072 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.066+0000 7f339633c900 20 enqueued request req=0x556b3ccf24a0 2026-03-20T11:46:17.072 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.066+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.072 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.066+0000 7f339633c900 20 req: 0x556b3ccf24a0 2026-03-20T11:46:17.072 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.066+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is io blocked 2026-03-20T11:46:17.072 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.066+0000 7f334cff9640 20 dequeued request req=0x556b3ccf24a0 2026-03-20T11:46:17.072 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.066+0000 7f334cff9640 20 RGWWQ: empty 2026-03-20T11:46:17.073 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.066+0000 7f334cff9640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 : miss 2026-03-20T11:46:17.073 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.066+0000 7f334cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.073 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f334cff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.073 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f334cff9640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 info.flags=0x17 2026-03-20T11:46:17.073 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f334cff9640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 to cache LRU end 2026-03-20T11:46:17.073 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f334cff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.073 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f334cff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.073 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f334cff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:17.073 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f334cff9640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-39 : miss 2026-03-20T11:46:17.073 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f334cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.073 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.067+0000 7f76ff4f0640 -1 req 9315308440381389789 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.073 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f334cff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.073 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f334cff9640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-39 info.flags=0x0 2026-03-20T11:46:17.073 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f334cff9640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-39 to cache LRU end 2026-03-20T11:46:17.073 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f334cff9640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-39 : miss 2026-03-20T11:46:17.073 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f334cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.073 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f334cff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.073 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f334cff9640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-39 info.flags=0x0 2026-03-20T11:46:17.073 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f334cff9640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-39 to cache LRU end 2026-03-20T11:46:17.073 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f334cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding source pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f334cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding dest pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f334cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f334cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f334cff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd30570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd30570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd30570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd30570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3cd23290:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3cd20340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3cd20340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = c3579208a6dbad6290f3c1c181cd75edbd4a585ace259a9361d8ee7c8d95d857 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:c3579208a6dbad6290f3c1c181cd75edbd4a585ace259a9361d8ee7c8d95d857 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 26ca2cdad7aa5f1aedf202653e8d92d850094b9c994e5f3a2d3555a436cf8980 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=26ca2cdad7aa5f1aedf202653e8d92d850094b9c994e5f3a2d3555a436cf8980 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=6, curl_handle=0x556b3cc93bb0 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.067+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is io blocked 2026-03-20T11:46:17.074 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.068+0000 7f32f1ffb640 20 link_request req_data=0x7f3334005bb0 req_data->id=6, curl_handle=0x556b3cc93bb0 2026-03-20T11:46:17.075 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.069+0000 7f7650392640 -1 req 12595722575024917014 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.075 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3cd20340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.075 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3cd20340:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.075 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3cd23290:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.075 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f339633c900 15 stack 0x556b3cd2fd80 end 2026-03-20T11:46:17.075 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is done 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f339633c900 20 collect(): s=0x556b3cbe9e50 stack=0x556b3cd2fd80 is complete 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f339633c900 15 stack 0x556b3cbe9e50 end 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is done 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f339633c900 4 starting trim on bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd30570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd30570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f339633c900 20 enqueued request req=0x556b3ccf24a0 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f339633c900 20 req: 0x556b3ccf24a0 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is io blocked 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f334c7f8640 20 dequeued request req=0x556b3ccf24a0 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f334c7f8640 20 RGWWQ: empty 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f334c7f8640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 : miss 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f334c7f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f334c7f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f334c7f8640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 info.flags=0x17 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f334c7f8640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 to cache LRU end 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f334c7f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f334c7f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f334c7f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f334c7f8640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-8 : miss 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f334c7f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f334c7f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f334c7f8640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-8 info.flags=0x0 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f334c7f8640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-8 to cache LRU end 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f334c7f8640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-8 : miss 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.070+0000 7f334c7f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f334c7f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f334c7f8640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-8 info.flags=0x0 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f334c7f8640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-8 to cache LRU end 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f334c7f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding source pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f334c7f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding dest pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f334c7f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f334c7f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f334c7f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd30570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd30570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd30570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.076 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd30570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3cd23290:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3cd20340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3cd20340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = ef3397df8a7ddfb4b7cd491ca03d45da36ce89d756c08bbc4154d59eb2c9ea8b 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:ef3397df8a7ddfb4b7cd491ca03d45da36ce89d756c08bbc4154d59eb2c9ea8b 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 55fbac252109f0183b0cf31f634cbc6a732fb7468481874e41c5a5e9608e8991 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=55fbac252109f0183b0cf31f634cbc6a732fb7468481874e41c5a5e9608e8991 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=7, curl_handle=0x556b3cc93bb0 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is io blocked 2026-03-20T11:46:17.077 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.071+0000 7f32f1ffb640 20 link_request req_data=0x7f3334005bb0 req_data->id=7, curl_handle=0x556b3cc93bb0 2026-03-20T11:46:17.078 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.072+0000 7f7652b97640 -1 req 18120940541476409018 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.096 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3ccd76f0:op=0x556b3ccea810:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3ccd76f0:op=0x556b3ccea810:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3ccd76f0:op=0x556b3ccea810:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3ccd76f0:op=0x556b3ccea810:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3ccd76f0:op=0x556b3ccea1c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 15 stack 0x556b3ccd76f0 end 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 run: stack=0x556b3ccd76f0 is done 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3cbe9a60:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 collect(): s=0x556b3cbe9a60 stack=0x556b3ccd76f0 is complete 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 10 trimming bilogs for bucket=:dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) markers=[00000000002.32.1,00000000002.28.1,00000000002.50.1,00000000002.28.1,00000000002.28.1,00000000002.47.1,00000000002.33.1,00000000002.30.1,00000000002.39.1,00000000002.57.1,00000000002.22.1], shards=11 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3cbe9a60:op=0x556b3cd1bfd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 10 trimming bilog shard 0 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.32.1 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 10 trimming bilog shard 1 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.28.1 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 10 trimming bilog shard 2 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.50.1 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 10 trimming bilog shard 3 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.28.1 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 10 trimming bilog shard 4 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.28.1 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 10 trimming bilog shard 5 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.47.1 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 10 trimming bilog shard 6 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.33.1 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 10 trimming bilog shard 7 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.30.1 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 10 trimming bilog shard 8 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.39.1 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 10 trimming bilog shard 9 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.57.1 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 10 trimming bilog shard 10 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.22.1 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 run: stack=0x556b3cbe9a60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3ccd76f0:op=0x556b3cd31330:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3c88b1d0:op=0x556b3cd21b60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3cd1c9c0:op=0x556b3cceff50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3ccd2070:op=0x556b3cd22650:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd1e580:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3ccf1420:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd27040:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd27d10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3cd38bc0:op=0x556b3cd25070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3cd37ee0:op=0x556b3ccea810:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3cd387a0:op=0x556b3cce6510:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3ccd76f0:op=0x556b3cd31330:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.0 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 run: stack=0x556b3ccd76f0 is io blocked 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3c88b1d0:op=0x556b3cd21b60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.1 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 run: stack=0x556b3c88b1d0 is io blocked 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3cd1c9c0:op=0x556b3cceff50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.2 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 run: stack=0x556b3cd1c9c0 is io blocked 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3ccd2070:op=0x556b3cd22650:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.3 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 run: stack=0x556b3ccd2070 is io blocked 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd1e580:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.097 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.4 2026-03-20T11:46:17.098 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 run: stack=0x556b3ccf26b0 is io blocked 2026-03-20T11:46:17.098 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3ccf1420:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.098 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.5 2026-03-20T11:46:17.098 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 run: stack=0x556b3ccee1e0 is io blocked 2026-03-20T11:46:17.098 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd27040:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.098 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.6 2026-03-20T11:46:17.098 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 run: stack=0x556b3cd2ffa0 is io blocked 2026-03-20T11:46:17.098 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd27d10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.098 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.7 2026-03-20T11:46:17.098 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 run: stack=0x556b3cd26450 is io blocked 2026-03-20T11:46:17.098 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3cd38bc0:op=0x556b3cd25070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.098 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.8 2026-03-20T11:46:17.098 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 run: stack=0x556b3cd38bc0 is io blocked 2026-03-20T11:46:17.098 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3cd37ee0:op=0x556b3ccea810:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.098 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.9 2026-03-20T11:46:17.098 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 run: stack=0x556b3cd37ee0 is io blocked 2026-03-20T11:46:17.098 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 cr:s=0x556b3cd387a0:op=0x556b3cce6510:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.098 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.10 2026-03-20T11:46:17.098 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.098+0000 7f339633c900 20 run: stack=0x556b3cd387a0 is io blocked 2026-03-20T11:46:17.099 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.100+0000 7f339633c900 20 cr:s=0x556b3cd38bc0:op=0x556b3cd25070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.099 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.100+0000 7f339633c900 20 cr:s=0x556b3cd38bc0:op=0x556b3cd25070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.099 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.100+0000 7f339633c900 20 cr:s=0x556b3cd38bc0:op=0x556b3cd25070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.099 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.100+0000 7f339633c900 20 cr:s=0x556b3cd38bc0:op=0x556b3cd25070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.099 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.100+0000 7f339633c900 15 stack 0x556b3cd38bc0 end 2026-03-20T11:46:17.099 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.100+0000 7f339633c900 20 run: stack=0x556b3cd38bc0 is done 2026-03-20T11:46:17.099 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.100+0000 7f339633c900 20 cr:s=0x556b3cbe9a60:op=0x556b3cd1bfd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:17.099 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.100+0000 7f339633c900 20 run: stack=0x556b3cbe9a60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.099 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.100+0000 7f339633c900 20 cr:s=0x556b3cd1c9c0:op=0x556b3cceff50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.099 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.100+0000 7f339633c900 20 cr:s=0x556b3cd1c9c0:op=0x556b3cceff50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.099 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.100+0000 7f339633c900 20 cr:s=0x556b3cd1c9c0:op=0x556b3cceff50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.099 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.100+0000 7f339633c900 20 cr:s=0x556b3cd1c9c0:op=0x556b3cceff50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.099 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.100+0000 7f339633c900 15 stack 0x556b3cd1c9c0 end 2026-03-20T11:46:17.099 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.100+0000 7f339633c900 20 run: stack=0x556b3cd1c9c0 is done 2026-03-20T11:46:17.099 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.100+0000 7f339633c900 20 cr:s=0x556b3cbe9a60:op=0x556b3cd1bfd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:17.099 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.100+0000 7f339633c900 20 run: stack=0x556b3cbe9a60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3ccd2070:op=0x556b3cd22650:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3ccd2070:op=0x556b3cd22650:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3ccd2070:op=0x556b3cd22650:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3ccd2070:op=0x556b3cd22650:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 15 stack 0x556b3ccd2070 end 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 run: stack=0x556b3ccd2070 is done 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cbe9a60:op=0x556b3cd1bfd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 run: stack=0x556b3cbe9a60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd1e580:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd1e580:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd1e580:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd1e580:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 15 stack 0x556b3ccf26b0 end 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 run: stack=0x556b3ccf26b0 is done 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cbe9a60:op=0x556b3cd1bfd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 run: stack=0x556b3cbe9a60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd27040:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd27040:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3ccf1420:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd27040:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3ccf1420:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd27040:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 15 stack 0x556b3cd2ffa0 end 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 run: stack=0x556b3cd2ffa0 is done 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3ccf1420:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cbe9a60:op=0x556b3cd1bfd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 run: stack=0x556b3cbe9a60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3ccf1420:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 15 stack 0x556b3ccee1e0 end 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 run: stack=0x556b3ccee1e0 is done 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cbe9a60:op=0x556b3cd1bfd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 run: stack=0x556b3cbe9a60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd27d10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd27d10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cd387a0:op=0x556b3cce6510:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd27d10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cd387a0:op=0x556b3cce6510:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd27d10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 15 stack 0x556b3cd26450 end 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 run: stack=0x556b3cd26450 is done 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cd387a0:op=0x556b3cce6510:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cbe9a60:op=0x556b3cd1bfd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 run: stack=0x556b3cbe9a60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cd387a0:op=0x556b3cce6510:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 15 stack 0x556b3cd387a0 end 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 run: stack=0x556b3cd387a0 is done 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cbe9a60:op=0x556b3cd1bfd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 run: stack=0x556b3cbe9a60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3c88b1d0:op=0x556b3cd21b60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3c88b1d0:op=0x556b3cd21b60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3c88b1d0:op=0x556b3cd21b60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3c88b1d0:op=0x556b3cd21b60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 15 stack 0x556b3c88b1d0 end 2026-03-20T11:46:17.100 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 run: stack=0x556b3c88b1d0 is done 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cbe9a60:op=0x556b3cd1bfd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 run: stack=0x556b3cbe9a60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3ccd76f0:op=0x556b3cd31330:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3ccd76f0:op=0x556b3cd31330:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3ccd76f0:op=0x556b3cd31330:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3ccd76f0:op=0x556b3cd31330:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 15 stack 0x556b3ccd76f0 end 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 run: stack=0x556b3ccd76f0 is done 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cbe9a60:op=0x556b3cd1bfd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 run: stack=0x556b3cbe9a60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cd37ee0:op=0x556b3ccea810:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cd37ee0:op=0x556b3ccea810:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cd37ee0:op=0x556b3ccea810:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cd37ee0:op=0x556b3ccea810:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 15 stack 0x556b3cd37ee0 end 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 run: stack=0x556b3cd37ee0 is done 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cbe9a60:op=0x556b3cd1bfd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cbe9a60:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 trim: trimmed bucket instance dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 15 stack 0x556b3cbe9a60 end 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 run: stack=0x556b3cbe9a60 is done 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 4 starting trim on bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd76f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd76f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 enqueued request req=0x556b3c887640 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 req: 0x556b3c887640 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f339633c900 20 run: stack=0x556b3cd26450 is io blocked 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f334b7f6640 20 dequeued request req=0x556b3c887640 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f334b7f6640 20 RGWWQ: empty 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f334b7f6640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 : miss 2026-03-20T11:46:17.101 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.101+0000 7f334b7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.102+0000 7f334b7f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.102+0000 7f334b7f6640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 info.flags=0x17 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.102+0000 7f334b7f6640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 to cache LRU end 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.102+0000 7f334b7f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.102+0000 7f334b7f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.102+0000 7f334b7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.102+0000 7f334b7f6640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-16 : miss 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.102+0000 7f334b7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.102+0000 7f334b7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.102+0000 7f334b7f6640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-16 info.flags=0x0 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.102+0000 7f334b7f6640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-16 to cache LRU end 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.102+0000 7f334b7f6640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-16 : miss 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.102+0000 7f334b7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f334b7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f334b7f6640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-16 info.flags=0x0 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f334b7f6640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-16 to cache LRU end 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f334b7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding source pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f334b7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding dest pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f334b7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f334b7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f334b7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd76f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd76f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd76f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd76f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3ccd76f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 20 run: stack=0x556b3cd26450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceff50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceff50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.102 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.103 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 341c7408feeba12d88eca8bf0a3aac72eef822c6c7797ccb8a539ea0eeac89d8 2026-03-20T11:46:17.103 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.103 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.103 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.103 INFO:teuthology.orchestra.run.vm07.stderr:341c7408feeba12d88eca8bf0a3aac72eef822c6c7797ccb8a539ea0eeac89d8 2026-03-20T11:46:17.103 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.103 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.103 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.103 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.103 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 69397ce60f38f2dae481292b665a67bca0cbbe5495ada2f44c46222cf5943137 2026-03-20T11:46:17.103 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=69397ce60f38f2dae481292b665a67bca0cbbe5495ada2f44c46222cf5943137 2026-03-20T11:46:17.103 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.103 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.103 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.103 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=8, curl_handle=0x556b3cce8410 2026-03-20T11:46:17.103 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f339633c900 20 run: stack=0x556b3ccee1e0 is io blocked 2026-03-20T11:46:17.103 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.103+0000 7f32f1ffb640 20 link_request req_data=0x556b3ccecae0 req_data->id=8, curl_handle=0x556b3cce8410 2026-03-20T11:46:17.103 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.105+0000 7f7686bff640 -1 req 13119362481952609887 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3ccf2980:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f339633c900 15 stack 0x556b3cce7430 end 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f339633c900 20 run: stack=0x556b3cce7430 is done 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f339633c900 20 collect(): s=0x556b3ccd07e0 stack=0x556b3cce7430 is complete 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f339633c900 15 stack 0x556b3ccd07e0 end 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is done 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f339633c900 4 starting trim on bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccf2810:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccf2810:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f339633c900 20 enqueued request req=0x556b3c88a060 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f339633c900 20 req: 0x556b3c88a060 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is io blocked 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f334aff5640 20 dequeued request req=0x556b3c88a060 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f334aff5640 20 RGWWQ: empty 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f334aff5640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 : miss 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f334aff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f334aff5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f334aff5640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 info.flags=0x17 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f334aff5640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 to cache LRU end 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f334aff5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f334aff5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.104 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f334aff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:17.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f334aff5640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-43 : miss 2026-03-20T11:46:17.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f334aff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.106+0000 7f334aff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f334aff5640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-43 info.flags=0x0 2026-03-20T11:46:17.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f334aff5640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-43 to cache LRU end 2026-03-20T11:46:17.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f334aff5640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-43 : miss 2026-03-20T11:46:17.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f334aff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f334aff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f334aff5640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-43 info.flags=0x0 2026-03-20T11:46:17.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f334aff5640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-43 to cache LRU end 2026-03-20T11:46:17.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f334aff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding source pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f334aff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding dest pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f334aff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f334aff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f334aff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:17.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccf2810:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccf2810:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccf2810:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccf2810:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cced030:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3ccf2810:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3ccf2810:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 780b271031a91fb644fb89e051b0a04da18a24a9be9846603c26c25ba7c4cb4f 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:780b271031a91fb644fb89e051b0a04da18a24a9be9846603c26c25ba7c4cb4f 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 468d11cf77a80209d4d0bd9d2df0ac61a585ab69f84a11e65345b4e383e4dad0 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=468d11cf77a80209d4d0bd9d2df0ac61a585ab69f84a11e65345b4e383e4dad0 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=9, curl_handle=0x556b3cce44b0 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f339633c900 20 run: stack=0x556b3cce7430 is io blocked 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.107+0000 7f32f1ffb640 20 link_request req_data=0x556b3cce4bc0 req_data->id=9, curl_handle=0x556b3cce44b0 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3cd240e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3cd240e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3c88ab90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.106 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f339633c900 15 stack 0x556b3cced850 end 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f339633c900 20 run: stack=0x556b3cced850 is done 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f339633c900 20 collect(): s=0x556b3cd15360 stack=0x556b3cced850 is complete 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f339633c900 15 stack 0x556b3cd15360 end 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f339633c900 20 run: stack=0x556b3cd15360 is done 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f339633c900 4 starting trim on bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccef210:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccef210:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f339633c900 20 enqueued request req=0x556b3cced660 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f339633c900 20 req: 0x556b3cced660 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f339633c900 20 run: stack=0x556b3cd15360 is io blocked 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f334bff7640 20 dequeued request req=0x556b3cced660 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f334bff7640 20 RGWWQ: empty 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f334bff7640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 : miss 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f334bff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f334bff7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f334bff7640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 info.flags=0x17 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f334bff7640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 to cache LRU end 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f334bff7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f334bff7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f334bff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f334bff7640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-10 : miss 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f334bff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f334bff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f334bff7640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-10 info.flags=0x0 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f334bff7640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-10 to cache LRU end 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f334bff7640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-10 : miss 2026-03-20T11:46:17.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.108+0000 7f334bff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.107 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.109+0000 7f7679be5640 -1 req 10890335107603568547 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f334bff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f334bff7640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-10 info.flags=0x0 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f334bff7640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-10 to cache LRU end 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f334bff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding source pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f334bff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding dest pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f334bff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f334bff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f334bff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccef210:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccef210:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccef210:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccef210:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3c88ab90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 20 run: stack=0x556b3cd15360 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3cd240e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3cd240e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 1d44fcbed85ee1226d2126a704e877b8b18f42037cc4f4008c4bb5258a0e5b01 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:1d44fcbed85ee1226d2126a704e877b8b18f42037cc4f4008c4bb5258a0e5b01 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 857e7aaa0af193feaa30b5446abbe98d78d9aae1a2eec0eec4ab5ca330c150fb 2026-03-20T11:46:17.109 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=857e7aaa0af193feaa30b5446abbe98d78d9aae1a2eec0eec4ab5ca330c150fb 2026-03-20T11:46:17.109 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.109 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.109 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.109 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=10, curl_handle=0x556b3c889a20 2026-03-20T11:46:17.109 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f339633c900 20 run: stack=0x556b3cced850 is io blocked 2026-03-20T11:46:17.109 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.109+0000 7f32f1ffb640 20 link_request req_data=0x556b3c88a5f0 req_data->id=10, curl_handle=0x556b3c889a20 2026-03-20T11:46:17.109 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.110+0000 7f769641e640 -1 req 15165963850369035858 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3cd20340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3cd20340:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3cd23290:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 15 stack 0x556b3cd2fd80 end 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is done 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 20 collect(): s=0x556b3cbe9e50 stack=0x556b3cd2fd80 is complete 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 15 stack 0x556b3cbe9e50 end 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is done 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 4 starting trim on bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd20b10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd20b10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 20 enqueued request req=0x556b3ccf24a0 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 20 req: 0x556b3ccf24a0 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is io blocked 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f334a7f4640 20 dequeued request req=0x556b3ccf24a0 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f334a7f4640 20 RGWWQ: empty 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f334a7f4640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 : miss 2026-03-20T11:46:17.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f334a7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.112 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f334a7f4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:46:17.112 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f334a7f4640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 info.flags=0x17 2026-03-20T11:46:17.112 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f334a7f4640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 to cache LRU end 2026-03-20T11:46:17.112 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f334a7f4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.112 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f334a7f4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.112 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f334a7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:17.112 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f334a7f4640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-3 : miss 2026-03-20T11:46:17.112 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f334a7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.112 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f334a7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.112 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f334a7f4640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-3 info.flags=0x0 2026-03-20T11:46:17.112 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f334a7f4640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-3 to cache LRU end 2026-03-20T11:46:17.112 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f334a7f4640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-3 : miss 2026-03-20T11:46:17.112 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f334a7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.112 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f334a7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.112 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f334a7f4640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-3 info.flags=0x0 2026-03-20T11:46:17.112 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f334a7f4640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-3 to cache LRU end 2026-03-20T11:46:17.112 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f334a7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding source pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.112 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f334a7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding dest pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.112 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f334a7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.112 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f334a7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.112 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f334a7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:17.112 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd20b10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.112 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd20b10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.112 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd20b10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd20b10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.113+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.114+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3cd20340:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.114+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.114+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.114+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3cd23290:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.114+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3cd23290:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.114+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.114+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.114+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.114+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.114+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.114+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.114+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 67bd518574a1e4404eaf98d595c4539d4ae28e42656147c2609f9b217861580a 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.114+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:67bd518574a1e4404eaf98d595c4539d4ae28e42656147c2609f9b217861580a 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.114+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.114+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.114+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.114+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.114+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = b037fcee37686e4d68a33e0060043961f38f84161f0344e923c3677e0554e080 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.114+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b037fcee37686e4d68a33e0060043961f38f84161f0344e923c3677e0554e080 2026-03-20T11:46:17.113 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.114+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.114 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.114+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.114 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.114+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.114 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.114+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=11, curl_handle=0x556b3cc93bb0 2026-03-20T11:46:17.114 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.114+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is io blocked 2026-03-20T11:46:17.114 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.114+0000 7f32f1ffb640 20 link_request req_data=0x7f3334005bb0 req_data->id=11, curl_handle=0x556b3cc93bb0 2026-03-20T11:46:17.114 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.115+0000 7f765439a640 -1 req 5484736627272269937 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.144 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceff50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.144 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceff50:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.144 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3ccd76f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.144 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f339633c900 15 stack 0x556b3ccee1e0 end 2026-03-20T11:46:17.144 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f339633c900 20 run: stack=0x556b3ccee1e0 is done 2026-03-20T11:46:17.144 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.144 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f339633c900 20 collect(): s=0x556b3cd26450 stack=0x556b3ccee1e0 is complete 2026-03-20T11:46:17.144 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.144 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.144 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.144 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.144 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f339633c900 15 stack 0x556b3cd26450 end 2026-03-20T11:46:17.144 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.144 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f339633c900 20 run: stack=0x556b3cd26450 is done 2026-03-20T11:46:17.144 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.144 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.145 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.145 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.145 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f339633c900 4 starting trim on bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:17.145 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd76a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.145 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd76a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.145 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f339633c900 20 enqueued request req=0x556b3c887640 2026-03-20T11:46:17.145 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.145 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f339633c900 20 req: 0x556b3c887640 2026-03-20T11:46:17.145 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f339633c900 20 run: stack=0x556b3cd26450 is io blocked 2026-03-20T11:46:17.145 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f3349ff3640 20 dequeued request req=0x556b3c887640 2026-03-20T11:46:17.145 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f3349ff3640 20 RGWWQ: empty 2026-03-20T11:46:17.145 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f3349ff3640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 : miss 2026-03-20T11:46:17.145 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f3349ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.145 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f3349ff3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:46:17.145 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f3349ff3640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 info.flags=0x17 2026-03-20T11:46:17.145 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f3349ff3640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 to cache LRU end 2026-03-20T11:46:17.145 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f3349ff3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.145 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f3349ff3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.145 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f3349ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:17.145 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f3349ff3640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-9 : miss 2026-03-20T11:46:17.145 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.146+0000 7f3349ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f3349ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f3349ff3640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-9 info.flags=0x0 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f3349ff3640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-9 to cache LRU end 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f3349ff3640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-9 : miss 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f3349ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f3349ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f3349ff3640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-9 info.flags=0x0 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f3349ff3640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-9 to cache LRU end 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f3349ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding source pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f3349ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding dest pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f3349ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f3349ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f3349ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd76a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd76a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd76a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd76a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3ccd76a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 20 run: stack=0x556b3cd26450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceff50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceff50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 6d1c2fe7550eb2ecca02f180f5d940d6ff6792b56ab0442fd0f70250f240033d 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.146 INFO:teuthology.orchestra.run.vm07.stderr:6d1c2fe7550eb2ecca02f180f5d940d6ff6792b56ab0442fd0f70250f240033d 2026-03-20T11:46:17.147 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.147 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.147 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.147 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.147 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 5843cc4b4c5620a10210e0851523da086211382327fc220148b733c7d5b70a9c 2026-03-20T11:46:17.147 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5843cc4b4c5620a10210e0851523da086211382327fc220148b733c7d5b70a9c 2026-03-20T11:46:17.147 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.147 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.147 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.147 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=12, curl_handle=0x556b3cce8410 2026-03-20T11:46:17.147 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f339633c900 20 run: stack=0x556b3ccee1e0 is io blocked 2026-03-20T11:46:17.147 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.147+0000 7f32f1ffb640 20 link_request req_data=0x556b3ccecae0 req_data->id=12, curl_handle=0x556b3cce8410 2026-03-20T11:46:17.147 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.149+0000 7f7673bd9640 -1 req 4995829361873299180 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.149 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3ccf2810:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.149 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3ccf2810:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.149 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cced030:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.149 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 15 stack 0x556b3cce7430 end 2026-03-20T11:46:17.149 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 run: stack=0x556b3cce7430 is done 2026-03-20T11:46:17.149 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3cd240e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.149 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3cd240e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.149 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.149 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 collect(): s=0x556b3ccd07e0 stack=0x556b3cce7430 is complete 2026-03-20T11:46:17.149 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.149 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.149 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.149 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.149 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 15 stack 0x556b3ccd07e0 end 2026-03-20T11:46:17.149 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.149 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is done 2026-03-20T11:46:17.149 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3c88ab90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.149 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 15 stack 0x556b3cced850 end 2026-03-20T11:46:17.149 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 run: stack=0x556b3cced850 is done 2026-03-20T11:46:17.149 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.149 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.149 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.149 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 collect(): s=0x556b3cd15360 stack=0x556b3cced850 is complete 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 cr:s=0x556b3cd15360:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 15 stack 0x556b3cd15360 end 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 run: stack=0x556b3cd15360 is done 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 4 starting trim on bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccf27f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 4 starting trim on bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccf27f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 enqueued request req=0x556b3ccf22b0 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 req: 0x556b3ccf22b0 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is io blocked 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 enqueued request req=0x556b3ccd84d0 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 req: 0x556b3ccf22b0 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 req: 0x556b3ccd84d0 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f339633c900 20 run: stack=0x556b3cced850 is io blocked 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f3348ff1640 20 dequeued request req=0x556b3ccf22b0 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f3348ff1640 20 RGWWQ: 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f3348ff1640 20 req: 0x556b3ccd84d0 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f33497f2640 20 dequeued request req=0x556b3ccd84d0 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f33497f2640 20 RGWWQ: empty 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f3348ff1640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 : miss 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f3348ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f33497f2640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 : miss 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f33497f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f33497f2640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.150 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f33497f2640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 info.flags=0x17 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f33497f2640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 to cache LRU end 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f33497f2640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f33497f2640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f3348ff1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f3348ff1640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 info.flags=0x17 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f3348ff1640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 to cache LRU end 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f3348ff1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f3348ff1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f3348ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f33497f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f3348ff1640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-38 : miss 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f33497f2640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-15 : miss 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f3348ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f33497f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f33497f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f33497f2640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-15 info.flags=0x0 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f33497f2640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-15 to cache LRU end 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f33497f2640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-15 : miss 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.151+0000 7f33497f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.152+0000 7f3348ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.152+0000 7f3348ff1640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-38 info.flags=0x0 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.152+0000 7f3348ff1640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-38 to cache LRU end 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.152+0000 7f3348ff1640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-38 : miss 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.152+0000 7f3348ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.152+0000 7f33497f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.152+0000 7f33497f2640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-15 info.flags=0x0 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.152+0000 7f33497f2640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-15 to cache LRU end 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.152+0000 7f33497f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding source pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.152+0000 7f33497f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding dest pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.152+0000 7f33497f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.152+0000 7f33497f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.152+0000 7f33497f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.152+0000 7f3348ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.152+0000 7f3348ff1640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-38 info.flags=0x0 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.152+0000 7f3348ff1640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-38 to cache LRU end 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.152+0000 7f3348ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding source pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.152+0000 7f3348ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding dest pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.152+0000 7f3348ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.152+0000 7f3348ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.151 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.152+0000 7f3348ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccf27f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3cd23290:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3cd23290:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccf27f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3cd20340:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 15 stack 0x556b3cd2fd80 end 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is done 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccf27f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 collect(): s=0x556b3cbe9e50 stack=0x556b3cd2fd80 is complete 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 15 stack 0x556b3cbe9e50 end 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is done 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccf27f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3c88ab90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 run: stack=0x556b3cced850 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.155 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 4 starting trim on bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3cd240e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cced030:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccf27f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3cd240e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = cc7c5dd0c179ff4486b571a1aa749cc74f2cf647f9eb9f8e69b37c34eaa6b654 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:cc7c5dd0c179ff4486b571a1aa749cc74f2cf647f9eb9f8e69b37c34eaa6b654 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = b4341cd0daa684fdaac6a61be8c81aaa4772c4d1c59db8a29c3ae09bc04b7e23 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b4341cd0daa684fdaac6a61be8c81aaa4772c4d1c59db8a29c3ae09bc04b7e23 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=13, curl_handle=0x556b3cc93bb0 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is io blocked 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cd23290:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccf27f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 enqueued request req=0x556b3cd1c5c0 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 req: 0x556b3cd1c5c0 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is io blocked 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cd23290:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.156 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 2699294982974e39fcc8b893e6d80d129e4c0bc712fc5c9733bee530eff2dd55 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:2699294982974e39fcc8b893e6d80d129e4c0bc712fc5c9733bee530eff2dd55 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 70c3f6d90ffd973f2bfdbdb2e6931a54fe067a5039745029cf2d03c59d4d6acd 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=70c3f6d90ffd973f2bfdbdb2e6931a54fe067a5039745029cf2d03c59d4d6acd 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=14, curl_handle=0x556b3c889a20 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f339633c900 20 run: stack=0x556b3cce7430 is io blocked 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f33487f0640 20 dequeued request req=0x556b3cd1c5c0 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f33487f0640 20 RGWWQ: empty 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.156+0000 7f33487f0640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 : miss 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.157+0000 7f33487f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.157+0000 7f32f1ffb640 20 link_request req_data=0x7f3334005bb0 req_data->id=13, curl_handle=0x556b3cc93bb0 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.157+0000 7f32f1ffb640 20 link_request req_data=0x556b3c88a5f0 req_data->id=14, curl_handle=0x556b3c889a20 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.157+0000 7f33487f0640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.157+0000 7f33487f0640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 info.flags=0x17 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.157+0000 7f33487f0640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 to cache LRU end 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.157+0000 7f33487f0640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.157+0000 7f33487f0640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.157 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.157+0000 7f33487f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.157+0000 7f33487f0640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-13 : miss 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.157+0000 7f33487f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.157+0000 7f33487f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.157+0000 7f33487f0640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-13 info.flags=0x0 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.157+0000 7f33487f0640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-13 to cache LRU end 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.157+0000 7f33487f0640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-13 : miss 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.157+0000 7f33487f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f33487f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f33487f0640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-13 info.flags=0x0 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f33487f0640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-13 to cache LRU end 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f33487f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18): adding source pipe: {s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f33487f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18): adding dest pipe: {s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f33487f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f33487f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f33487f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccf27f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccf27f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccf27f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccf27f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd37ee0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd20340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd20340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 1c13316220872bf8ef7d622cb0f5c3d81e395ac634d20e0abcbe922108dde754 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:1c13316220872bf8ef7d622cb0f5c3d81e395ac634d20e0abcbe922108dde754 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = f84ffa1ccf0f5ca4742ba91247869bab12131b0504d3908af2f5773eff28f1a5 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=f84ffa1ccf0f5ca4742ba91247869bab12131b0504d3908af2f5773eff28f1a5 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=15, curl_handle=0x556b3cce44b0 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f339633c900 20 run: stack=0x556b3cd2ffa0 is io blocked 2026-03-20T11:46:17.158 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.158+0000 7f32f1ffb640 20 link_request req_data=0x556b3cce4bc0 req_data->id=15, curl_handle=0x556b3cce44b0 2026-03-20T11:46:17.159 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.159+0000 7f7678be3640 -1 req 15254842372165275241 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.159 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.159+0000 7f7695c1d640 -1 req 5335279825691274467 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.159 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.159+0000 7f769a426640 -1 req 616247088663201927 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceff50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceff50:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3ccd76a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f339633c900 15 stack 0x556b3ccee1e0 end 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f339633c900 20 run: stack=0x556b3ccee1e0 is done 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f339633c900 20 collect(): s=0x556b3cd26450 stack=0x556b3ccee1e0 is complete 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f339633c900 15 stack 0x556b3cd26450 end 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f339633c900 20 run: stack=0x556b3cd26450 is done 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f339633c900 4 starting trim on bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccf27f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccf27f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f339633c900 20 enqueued request req=0x556b3c887640 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f339633c900 20 req: 0x556b3c887640 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f339633c900 20 run: stack=0x556b3cd26450 is io blocked 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f3347fef640 20 dequeued request req=0x556b3c887640 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f3347fef640 20 RGWWQ: empty 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f3347fef640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 : miss 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f3347fef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f3347fef640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f3347fef640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 info.flags=0x17 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f3347fef640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 to cache LRU end 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f3347fef640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f3347fef640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f3347fef640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f3347fef640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-11 : miss 2026-03-20T11:46:17.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f3347fef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f3347fef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f3347fef640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-11 info.flags=0x0 2026-03-20T11:46:17.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f3347fef640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-11 to cache LRU end 2026-03-20T11:46:17.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f3347fef640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-11 : miss 2026-03-20T11:46:17.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.190+0000 7f3347fef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f3347fef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f3347fef640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-11 info.flags=0x0 2026-03-20T11:46:17.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f3347fef640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-11 to cache LRU end 2026-03-20T11:46:17.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f3347fef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16): adding source pipe: {s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f3347fef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16): adding dest pipe: {s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f3347fef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f3347fef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f3347fef640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:17.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccf27f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccf27f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccf27f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccf27f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3ccf27f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 20 run: stack=0x556b3cd26450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cd1e580:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cd1e580:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 7def490f592fd0f9f77a582fc618952df7cbc07d7a8b8473f771059e7d626f54 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:7def490f592fd0f9f77a582fc618952df7cbc07d7a8b8473f771059e7d626f54 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = f8713c867af56b4b7ae36b813133edb3b4336e59d52f615d950d5c79f918eb9f 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=f8713c867af56b4b7ae36b813133edb3b4336e59d52f615d950d5c79f918eb9f 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=16, curl_handle=0x556b3cce8410 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f339633c900 20 run: stack=0x556b3ccee1e0 is io blocked 2026-03-20T11:46:17.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.192+0000 7f32f1ffb640 20 link_request req_data=0x556b3ccecae0 req_data->id=16, curl_handle=0x556b3cce8410 2026-03-20T11:46:17.191 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.193+0000 7f76ae44e640 -1 req 4072761571232138873 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cd23290:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cd23290:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cced030:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 15 stack 0x556b3cce7430 end 2026-03-20T11:46:17.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 run: stack=0x556b3cce7430 is done 2026-03-20T11:46:17.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd20340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd20340:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3cd240e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3cd240e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 collect(): s=0x556b3ccd07e0 stack=0x556b3cce7430 is complete 2026-03-20T11:46:17.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 15 stack 0x556b3ccd07e0 end 2026-03-20T11:46:17.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is done 2026-03-20T11:46:17.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd37ee0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 15 stack 0x556b3cd2ffa0 end 2026-03-20T11:46:17.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 run: stack=0x556b3cd2ffa0 is done 2026-03-20T11:46:17.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3c88ab90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 15 stack 0x556b3cd2fd80 end 2026-03-20T11:46:17.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is done 2026-03-20T11:46:17.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 collect(): s=0x556b3cbe9e50 stack=0x556b3cd2ffa0 is complete 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 15 stack 0x556b3cbe9e50 end 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is done 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 collect(): s=0x556b3cced850 stack=0x556b3cd2fd80 is complete 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 cr:s=0x556b3cced850:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 15 stack 0x556b3cced850 end 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 run: stack=0x556b3cced850 is done 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 4 starting trim on bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 4 starting trim on bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 enqueued request req=0x556b3ccd84d0 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 req: 0x556b3ccd84d0 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f33477ee640 20 dequeued request req=0x556b3ccd84d0 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f33477ee640 20 RGWWQ: empty 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f33477ee640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 : miss 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f33477ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is io blocked 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 4 starting trim on bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 enqueued request req=0x556b3ccd8830 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 req: 0x556b3ccd8830 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is io blocked 2026-03-20T11:46:17.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd20ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd20ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 enqueued request req=0x556b3cbe9c60 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 req: 0x556b3ccd8830 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 req: 0x556b3cbe9c60 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f33467ec640 20 dequeued request req=0x556b3ccd8830 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f33467ec640 20 RGWWQ: 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f33467ec640 20 req: 0x556b3cbe9c60 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f33467ec640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 : miss 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f33467ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is io blocked 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f3346fed640 20 dequeued request req=0x556b3cbe9c60 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f3346fed640 20 RGWWQ: empty 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f3346fed640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 : miss 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f3346fed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f33477ee640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f33477ee640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 info.flags=0x17 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f33477ee640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 to cache LRU end 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f33477ee640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f33477ee640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f33477ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f33477ee640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-12 : miss 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f33477ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f33467ec640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f33467ec640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 info.flags=0x17 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f33467ec640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 to cache LRU end 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f33467ec640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f33467ec640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f33467ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f33467ec640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-40 : miss 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.200+0000 7f33467ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f3346fed640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f3346fed640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 info.flags=0x17 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f3346fed640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 to cache LRU end 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f3346fed640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f3346fed640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f3346fed640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f3346fed640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-18 : miss 2026-03-20T11:46:17.201 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f3346fed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.202 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f33477ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.202 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f33477ee640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-12 info.flags=0x0 2026-03-20T11:46:17.202 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f33477ee640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-12 to cache LRU end 2026-03-20T11:46:17.202 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f33477ee640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-12 : miss 2026-03-20T11:46:17.202 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f33477ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.202 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f3346fed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.202 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f3346fed640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-18 info.flags=0x0 2026-03-20T11:46:17.202 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f3346fed640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-18 to cache LRU end 2026-03-20T11:46:17.202 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f33467ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f3346fed640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-18 : miss 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f33467ec640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-40 info.flags=0x0 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f33467ec640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-40 to cache LRU end 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f33467ec640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-40 : miss 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f33467ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f3346fed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f33477ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f33477ee640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-12 info.flags=0x0 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f33477ee640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-12 to cache LRU end 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f33477ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17): adding source pipe: {s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f33477ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17): adding dest pipe: {s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f33477ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f33477ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f33477ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd240e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 8cffc346e0dede4dcdfad285fe9b9c62f077998666a6d8839c9f786aba6acf1b 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:8cffc346e0dede4dcdfad285fe9b9c62f077998666a6d8839c9f786aba6acf1b 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = dabca2b7b6ed7f873a39b1b8abfc2ebbd9b5c75aa44d98ac02a8fd4a37ef6a98 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=dabca2b7b6ed7f873a39b1b8abfc2ebbd9b5c75aa44d98ac02a8fd4a37ef6a98 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=17, curl_handle=0x556b3cc93bb0 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f339633c900 20 run: stack=0x556b3cd2ffa0 is io blocked 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f32f1ffb640 20 link_request req_data=0x7f3334005bb0 req_data->id=17, curl_handle=0x556b3cc93bb0 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f33467ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.201+0000 7f33467ec640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-40 info.flags=0x0 2026-03-20T11:46:17.203 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f33467ec640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-40 to cache LRU end 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f33467ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1): adding source pipe: {s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f33467ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1): adding dest pipe: {s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f33467ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f33467ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f33467ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cd23290:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3c88ab90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3c88ab90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 70c1b7ae7d835dcc837cea1c93b0ef3e4bac41b8350b1738df5e64419774a3da 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:70c1b7ae7d835dcc837cea1c93b0ef3e4bac41b8350b1738df5e64419774a3da 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 19e26e5b7254ecf746bcb75619d1a06ae956da6a2ccc6fd7dea4f73ad447ed6f 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=19e26e5b7254ecf746bcb75619d1a06ae956da6a2ccc6fd7dea4f73ad447ed6f 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=18, curl_handle=0x556b3cce44b0 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 run: stack=0x556b3cce7430 is io blocked 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f32f1ffb640 20 link_request req_data=0x556b3cce4bc0 req_data->id=18, curl_handle=0x556b3cce44b0 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f3346fed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f3346fed640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-18 info.flags=0x0 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f3346fed640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-18 to cache LRU end 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f3346fed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23): adding source pipe: {s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f3346fed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23): adding dest pipe: {s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f3346fed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f3346fed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f3346fed640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:17.204 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd20ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd20ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd20ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd20ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd20340:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd20ad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd20ad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 30551409d5e1851a2090c4e30ab09763af547c0fe69df1af39eb7fd7fa4b920e 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:30551409d5e1851a2090c4e30ab09763af547c0fe69df1af39eb7fd7fa4b920e 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = ef83d0616e7ca4f1410c91f86c0f8e18ae363e43cffc0016f586a3475d14e6d9 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=ef83d0616e7ca4f1410c91f86c0f8e18ae363e43cffc0016f586a3475d14e6d9 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=19, curl_handle=0x556b3c889a20 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f339633c900 20 run: stack=0x556b3ccf26b0 is io blocked 2026-03-20T11:46:17.205 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.202+0000 7f32f1ffb640 20 link_request req_data=0x556b3c88a5f0 req_data->id=19, curl_handle=0x556b3c889a20 2026-03-20T11:46:17.205 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.203+0000 7f76b9c65640 -1 req 5749711356613045657 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.205 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.204+0000 7f76c1474640 -1 req 6485509932942113992 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.205 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.204+0000 7f76bd46c640 -1 req 14888415026982571043 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.232 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cd1e580:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.232 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cd1e580:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.232 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3ccf27f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.232 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f339633c900 15 stack 0x556b3ccee1e0 end 2026-03-20T11:46:17.232 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f339633c900 20 run: stack=0x556b3ccee1e0 is done 2026-03-20T11:46:17.232 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.232 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f339633c900 20 collect(): s=0x556b3cd26450 stack=0x556b3ccee1e0 is complete 2026-03-20T11:46:17.232 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.232 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.232 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.232 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.232 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f339633c900 15 stack 0x556b3cd26450 end 2026-03-20T11:46:17.232 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.232 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f339633c900 20 run: stack=0x556b3cd26450 is done 2026-03-20T11:46:17.232 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.232 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.232 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.232 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.232 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f339633c900 4 starting trim on bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:17.232 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cceee30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.232 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cceee30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.233 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f339633c900 20 enqueued request req=0x556b3c887640 2026-03-20T11:46:17.233 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.233 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f339633c900 20 req: 0x556b3c887640 2026-03-20T11:46:17.233 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f339633c900 20 run: stack=0x556b3cd26450 is io blocked 2026-03-20T11:46:17.233 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f3345feb640 20 dequeued request req=0x556b3c887640 2026-03-20T11:46:17.233 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f3345feb640 20 RGWWQ: empty 2026-03-20T11:46:17.233 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f3345feb640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 : miss 2026-03-20T11:46:17.233 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f3345feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.233 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f3345feb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.233 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f3345feb640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 info.flags=0x17 2026-03-20T11:46:17.233 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f3345feb640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 to cache LRU end 2026-03-20T11:46:17.233 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f3345feb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.233 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f3345feb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.233 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f3345feb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:17.233 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.234+0000 7f3345feb640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-50 : miss 2026-03-20T11:46:17.233 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f3345feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.233 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f3345feb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.233 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f3345feb640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-50 info.flags=0x0 2026-03-20T11:46:17.233 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f3345feb640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-50 to cache LRU end 2026-03-20T11:46:17.233 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f3345feb640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-50 : miss 2026-03-20T11:46:17.233 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f3345feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f3345feb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f3345feb640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-50 info.flags=0x0 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f3345feb640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-50 to cache LRU end 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f3345feb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10): adding source pipe: {s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f3345feb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10): adding dest pipe: {s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f3345feb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f3345feb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f3345feb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cceee30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cceee30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cceee30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cceee30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cd1e580:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 20 run: stack=0x556b3cd26450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceee30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceee30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 9840b7d46f2310b1d51fb56bc2ba3c9ecc53809170df0916905a94083ce56b30 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.234 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.235 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.235 INFO:teuthology.orchestra.run.vm07.stderr:9840b7d46f2310b1d51fb56bc2ba3c9ecc53809170df0916905a94083ce56b30 2026-03-20T11:46:17.235 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.235 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.235 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.235 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.235 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 53410d63f5029422d3f51172f7d08a2a372be5657b9064c8dabd0f11903425f4 2026-03-20T11:46:17.235 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=53410d63f5029422d3f51172f7d08a2a372be5657b9064c8dabd0f11903425f4 2026-03-20T11:46:17.235 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.235 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.235 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.235 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=20, curl_handle=0x556b3cce8410 2026-03-20T11:46:17.235 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f339633c900 20 run: stack=0x556b3ccee1e0 is io blocked 2026-03-20T11:46:17.235 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.235+0000 7f32f1ffb640 20 link_request req_data=0x556b3ccecae0 req_data->id=20, curl_handle=0x556b3cce8410 2026-03-20T11:46:17.235 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.237+0000 7f76e1cb5640 -1 req 11023299999064704105 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.242 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.242 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.242 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd240e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.242 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f339633c900 15 stack 0x556b3cd2ffa0 end 2026-03-20T11:46:17.242 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f339633c900 20 run: stack=0x556b3cd2ffa0 is done 2026-03-20T11:46:17.242 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.242 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f339633c900 20 collect(): s=0x556b3ccd07e0 stack=0x556b3cd2ffa0 is complete 2026-03-20T11:46:17.242 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.242 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.242 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.242 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.242 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f339633c900 15 stack 0x556b3ccd07e0 end 2026-03-20T11:46:17.242 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.242 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is done 2026-03-20T11:46:17.242 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.242 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.242 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.242 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.242 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f339633c900 4 starting trim on bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:17.242 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.242 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.242 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f339633c900 20 enqueued request req=0x556b3ccd84d0 2026-03-20T11:46:17.242 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.242 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f339633c900 20 req: 0x556b3ccd84d0 2026-03-20T11:46:17.243 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is io blocked 2026-03-20T11:46:17.243 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f33457ea640 20 dequeued request req=0x556b3ccd84d0 2026-03-20T11:46:17.243 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f33457ea640 20 RGWWQ: empty 2026-03-20T11:46:17.243 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f33457ea640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 : miss 2026-03-20T11:46:17.243 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f33457ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.243 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f33457ea640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.243 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f33457ea640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 info.flags=0x17 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f33457ea640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 to cache LRU end 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f33457ea640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f33457ea640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f33457ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f33457ea640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-24 : miss 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.244+0000 7f33457ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f33457ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f33457ea640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-24 info.flags=0x0 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f33457ea640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-24 to cache LRU end 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3c88ab90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f33457ea640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-24 : miss 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f33457ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3c88ab90:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cd23290:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 15 stack 0x556b3cce7430 end 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 run: stack=0x556b3cce7430 is done 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 collect(): s=0x556b3cd2fd80 stack=0x556b3cce7430 is complete 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 15 stack 0x556b3cd2fd80 end 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is done 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 4 starting trim on bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 enqueued request req=0x556b3ccd8830 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 req: 0x556b3ccd8830 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is io blocked 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f3344fe9640 20 dequeued request req=0x556b3ccd8830 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f3344fe9640 20 RGWWQ: empty 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f3344fe9640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 : miss 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f3344fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f33457ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f33457ea640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-24 info.flags=0x0 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f33457ea640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-24 to cache LRU end 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f33457ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29): adding source pipe: {s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f33457ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29): adding dest pipe: {s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f33457ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f33457ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f33457ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cd240e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 189da2c0404fdaa76013a06cef8bbeec56465aec42d0a9c3e60ebb3b6afc84fb 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:189da2c0404fdaa76013a06cef8bbeec56465aec42d0a9c3e60ebb3b6afc84fb 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.245 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.246 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 5423f2246bded0bc03bb6c3d14cce23339997d4e68cab0ffc2e26487293e1359 2026-03-20T11:46:17.246 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5423f2246bded0bc03bb6c3d14cce23339997d4e68cab0ffc2e26487293e1359 2026-03-20T11:46:17.246 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.246 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.246 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.246 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=21, curl_handle=0x556b3cce44b0 2026-03-20T11:46:17.246 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f339633c900 20 run: stack=0x556b3cce7430 is io blocked 2026-03-20T11:46:17.246 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f32f1ffb640 20 link_request req_data=0x556b3cce4bc0 req_data->id=21, curl_handle=0x556b3cce44b0 2026-03-20T11:46:17.246 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f3344fe9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.246 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f3344fe9640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 info.flags=0x17 2026-03-20T11:46:17.246 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f3344fe9640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 to cache LRU end 2026-03-20T11:46:17.246 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f3344fe9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.246 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f3344fe9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.246 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f3344fe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:46:17.246 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.245+0000 7f3344fe9640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-56 : miss 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd20ad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd20ad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd20340:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f339633c900 15 stack 0x556b3ccf26b0 end 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f339633c900 20 run: stack=0x556b3ccf26b0 is done 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f339633c900 20 collect(): s=0x556b3cbe9e50 stack=0x556b3ccf26b0 is complete 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f339633c900 15 stack 0x556b3cbe9e50 end 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is done 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f339633c900 4 starting trim on bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f339633c900 20 enqueued request req=0x556b3cbe9c60 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f339633c900 20 req: 0x556b3cbe9c60 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is io blocked 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f33447e8640 20 dequeued request req=0x556b3cbe9c60 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f33447e8640 20 RGWWQ: empty 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f33447e8640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 : miss 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f33447e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f3344fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f33447e8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f33447e8640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 info.flags=0x17 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f33447e8640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 to cache LRU end 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f33447e8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f33447e8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f33447e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f33447e8640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-27 : miss 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f33447e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f3344fe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f3344fe9640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-56 info.flags=0x0 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f3344fe9640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-56 to cache LRU end 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f3344fe9640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-56 : miss 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f3344fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f3344fe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f3344fe9640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-56 info.flags=0x0 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f3344fe9640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-56 to cache LRU end 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f3344fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16): adding source pipe: {s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f33447e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f33447e8640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-27 info.flags=0x0 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f33447e8640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-27 to cache LRU end 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f33447e8640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-27 : miss 2026-03-20T11:46:17.247 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f33447e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.246+0000 7f3344fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16): adding dest pipe: {s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f3344fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f3344fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f3344fe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3ccf27f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3c88ab90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3c88ab90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = f03de0ffc97f6441b607e76a20ea7462af2c3cdd5522c12261a645884732cfc2 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:f03de0ffc97f6441b607e76a20ea7462af2c3cdd5522c12261a645884732cfc2 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 7a62ebcfb200e012e16a8b011040c16fcbd5649b19b19c4dd2f358209c48f29f 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=7a62ebcfb200e012e16a8b011040c16fcbd5649b19b19c4dd2f358209c48f29f 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=22, curl_handle=0x556b3c889a20 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 run: stack=0x556b3ccf26b0 is io blocked 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f32f1ffb640 20 link_request req_data=0x556b3c88a5f0 req_data->id=22, curl_handle=0x556b3c889a20 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f33447e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f33447e8640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-27 info.flags=0x0 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f33447e8640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-27 to cache LRU end 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f33447e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32): adding source pipe: {s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f33447e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32): adding dest pipe: {s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f33447e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f33447e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f33447e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:46:17.248 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.247+0000 7f76b2456640 -1 req 1705333595224984509 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd23290:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd20ad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd20ad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = e50b02fd5b37128afba3424a8faf0ba77d2ff982b580e2b470016bb0af29c1c8 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:e50b02fd5b37128afba3424a8faf0ba77d2ff982b580e2b470016bb0af29c1c8 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 1634fa947474269686c43375c73be6bcbce53abfa7b6fc03107f4218c2d475d7 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=1634fa947474269686c43375c73be6bcbce53abfa7b6fc03107f4218c2d475d7 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=23, curl_handle=0x556b3cc93bb0 2026-03-20T11:46:17.249 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f339633c900 20 run: stack=0x556b3cd2ffa0 is io blocked 2026-03-20T11:46:17.250 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.247+0000 7f32f1ffb640 20 link_request req_data=0x7f3334005bb0 req_data->id=23, curl_handle=0x556b3cc93bb0 2026-03-20T11:46:17.250 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.248+0000 7f76d449a640 -1 req 6859130990175200911 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.250 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.249+0000 7f76e8cc3640 -1 req 101647065565463075 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.276 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceee30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.276 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceee30:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.276 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cd1e580:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.276 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f339633c900 15 stack 0x556b3ccee1e0 end 2026-03-20T11:46:17.276 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f339633c900 20 run: stack=0x556b3ccee1e0 is done 2026-03-20T11:46:17.276 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.276 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f339633c900 20 collect(): s=0x556b3cd26450 stack=0x556b3ccee1e0 is complete 2026-03-20T11:46:17.276 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.276 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.276 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.276 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.276 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f339633c900 15 stack 0x556b3cd26450 end 2026-03-20T11:46:17.276 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.276 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f339633c900 20 run: stack=0x556b3cd26450 is done 2026-03-20T11:46:17.276 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.276 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.276 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.276 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.277 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f339633c900 4 starting trim on bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:46:17.277 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cceee30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.277 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cceee30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.277 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f339633c900 20 enqueued request req=0x556b3c887640 2026-03-20T11:46:17.277 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.277 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f339633c900 20 req: 0x556b3c887640 2026-03-20T11:46:17.277 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f339633c900 20 run: stack=0x556b3cd26450 is io blocked 2026-03-20T11:46:17.277 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f3343fe7640 20 dequeued request req=0x556b3c887640 2026-03-20T11:46:17.277 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f3343fe7640 20 RGWWQ: empty 2026-03-20T11:46:17.277 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f3343fe7640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 : miss 2026-03-20T11:46:17.277 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.278+0000 7f3343fe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.277 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f3343fe7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:17.277 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f3343fe7640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 info.flags=0x17 2026-03-20T11:46:17.277 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f3343fe7640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 to cache LRU end 2026-03-20T11:46:17.277 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f3343fe7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.277 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f3343fe7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.277 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f3343fe7640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:46:17.277 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f3343fe7640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-44 : miss 2026-03-20T11:46:17.277 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f3343fe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.277 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f3343fe7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.277 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f3343fe7640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-44 info.flags=0x0 2026-03-20T11:46:17.277 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f3343fe7640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-44 to cache LRU end 2026-03-20T11:46:17.277 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f3343fe7640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-44 : miss 2026-03-20T11:46:17.277 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f3343fe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.277 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f3343fe7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.277 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f3343fe7640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-44 info.flags=0x0 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f3343fe7640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-44 to cache LRU end 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f3343fe7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4): adding source pipe: {s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f3343fe7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4): adding dest pipe: {s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f3343fe7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f3343fe7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f3343fe7640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cceee30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cceee30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cceee30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cceee30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cd20340:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 20 run: stack=0x556b3cd26450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceee30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceee30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 7352798d55ef2afb6a5961b14d7ba2845b2126dc59b85ba6f9e696f981b32283 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:7352798d55ef2afb6a5961b14d7ba2845b2126dc59b85ba6f9e696f981b32283 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = faeab2ff326a847d3731bca30560d09f8dfc98ad86a33180b4ebc61058d41621 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=faeab2ff326a847d3731bca30560d09f8dfc98ad86a33180b4ebc61058d41621 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=24, curl_handle=0x556b3cce8410 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f339633c900 20 run: stack=0x556b3ccee1e0 is io blocked 2026-03-20T11:46:17.278 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.279+0000 7f32f1ffb640 20 link_request req_data=0x556b3ccecae0 req_data->id=24, curl_handle=0x556b3cce8410 2026-03-20T11:46:17.279 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.281+0000 7f77004f2640 -1 req 16449569182730101528 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cd240e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f339633c900 15 stack 0x556b3cce7430 end 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f339633c900 20 run: stack=0x556b3cce7430 is done 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f339633c900 20 collect(): s=0x556b3ccd07e0 stack=0x556b3cce7430 is complete 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f339633c900 15 stack 0x556b3ccd07e0 end 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is done 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f339633c900 4 starting trim on bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f339633c900 20 enqueued request req=0x556b3ccd84d0 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f339633c900 20 req: 0x556b3ccd84d0 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is io blocked 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f33437e6640 20 dequeued request req=0x556b3ccd84d0 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f33437e6640 20 RGWWQ: empty 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f33437e6640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 : miss 2026-03-20T11:46:17.286 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f33437e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f33437e6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f33437e6640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 info.flags=0x17 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f33437e6640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 to cache LRU end 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f33437e6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f33437e6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f33437e6640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f33437e6640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-17 : miss 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f33437e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f33437e6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f33437e6640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-17 info.flags=0x0 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f33437e6640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-17 to cache LRU end 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f33437e6640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-17 : miss 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.288+0000 7f33437e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3c88ab90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3c88ab90:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3ccf27f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 15 stack 0x556b3ccf26b0 end 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 run: stack=0x556b3ccf26b0 is done 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 collect(): s=0x556b3cd2fd80 stack=0x556b3ccf26b0 is complete 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 15 stack 0x556b3cd2fd80 end 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is done 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 4 starting trim on bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 enqueued request req=0x556b3ccd8830 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 req: 0x556b3ccd8830 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is io blocked 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f33437e6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f33437e6640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-17 info.flags=0x0 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f33437e6640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-17 to cache LRU end 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f33437e6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22): adding source pipe: {s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f33437e6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22): adding dest pipe: {s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.287 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f33437e6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f33437e6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f33437e6640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f33437e6640 20 dequeued request req=0x556b3ccd8830 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f33437e6640 20 RGWWQ: empty 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f33437e6640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 : miss 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f33437e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd240e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 0221205d258b7f4bfc875c5b274e757caca8e0b65920fa4aa98f048706a86b07 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.288 INFO:teuthology.orchestra.run.vm07.stderr:0221205d258b7f4bfc875c5b274e757caca8e0b65920fa4aa98f048706a86b07 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = e8a7433246e755a7f2e2e75286ea63a271baeb8cbc3f7277726a642fb8e088dd 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=e8a7433246e755a7f2e2e75286ea63a271baeb8cbc3f7277726a642fb8e088dd 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=25, curl_handle=0x556b3c889a20 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f339633c900 20 run: stack=0x556b3ccf26b0 is io blocked 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f32f1ffb640 20 link_request req_data=0x556b3c88a5f0 req_data->id=25, curl_handle=0x556b3c889a20 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f33437e6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f33437e6640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 info.flags=0x17 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f33437e6640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 to cache LRU end 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f33437e6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f33437e6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f33437e6640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f33437e6640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f33437e6640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-46 : miss 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f33437e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f33437e6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f33437e6640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-46 info.flags=0x0 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f33437e6640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-46 to cache LRU end 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f33437e6640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-46 : miss 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.289+0000 7f33437e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd20ad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd20ad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd23290:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 15 stack 0x556b3cd2ffa0 end 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 run: stack=0x556b3cd2ffa0 is done 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 collect(): s=0x556b3cbe9e50 stack=0x556b3cd2ffa0 is complete 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 15 stack 0x556b3cbe9e50 end 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is done 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 4 starting trim on bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 enqueued request req=0x556b3cbe9c60 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 req: 0x556b3cbe9c60 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is io blocked 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f33427e4640 20 dequeued request req=0x556b3cbe9c60 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f33427e4640 20 RGWWQ: empty 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f33427e4640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 : miss 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f33427e4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f33437e6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f33437e6640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-46 info.flags=0x0 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f33437e6640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-46 to cache LRU end 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f33437e6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6): adding source pipe: {s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f33437e6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6): adding dest pipe: {s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f33437e6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f33437e6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f33437e6640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3c88ab90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd23290:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd23290:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = b9e0e8d2d12bc77149084f19e86adbc3e6a3a0ead64847e162a1054bec4e468c 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:b9e0e8d2d12bc77149084f19e86adbc3e6a3a0ead64847e162a1054bec4e468c 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = f1027ff7e2a1dcb929078f42ed1c52fd34b5ddd0e44b825237ef8e67f94d7c7c 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=f1027ff7e2a1dcb929078f42ed1c52fd34b5ddd0e44b825237ef8e67f94d7c7c 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=26, curl_handle=0x556b3cc93bb0 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f339633c900 20 run: stack=0x556b3cd2ffa0 is io blocked 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f32f1ffb640 20 link_request req_data=0x7f3334005bb0 req_data->id=26, curl_handle=0x556b3cc93bb0 2026-03-20T11:46:17.290 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.291+0000 7f76ed4cc640 -1 req 5570767709497694064 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.290+0000 7f33427e4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.290 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.291+0000 7f33427e4640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 info.flags=0x17 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.291+0000 7f33427e4640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 to cache LRU end 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.291+0000 7f33427e4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.291+0000 7f33427e4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.291+0000 7f33427e4640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.291+0000 7f33427e4640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-37 : miss 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.291+0000 7f33427e4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.291+0000 7f33427e4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.291+0000 7f33427e4640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-37 info.flags=0x0 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.291+0000 7f33427e4640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-37 to cache LRU end 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.291+0000 7f33427e4640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-37 : miss 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.291+0000 7f33427e4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.291 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.291+0000 7f7712d17640 -1 req 1594063805616604595 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f33427e4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f33427e4640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-37 info.flags=0x0 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f33427e4640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-37 to cache LRU end 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f33427e4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42): adding source pipe: {s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f33427e4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42): adding dest pipe: {s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f33427e4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f33427e4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f33427e4640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3c8881f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cd20ad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cd20ad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.291 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.292 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.292 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.292 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.292 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.292 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.292 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.292 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.292 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = dae3687dae250077425f57bea4bd08515dc5f94daf141089a9733a49763eefd7 2026-03-20T11:46:17.292 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.292 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.292 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.292 INFO:teuthology.orchestra.run.vm07.stderr:dae3687dae250077425f57bea4bd08515dc5f94daf141089a9733a49763eefd7 2026-03-20T11:46:17.292 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.292 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.292 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.292 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.292 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = e18456d4365b54b466c65e3e5ba627b6b45fd8ff055114a281781c328421e7bf 2026-03-20T11:46:17.292 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=e18456d4365b54b466c65e3e5ba627b6b45fd8ff055114a281781c328421e7bf 2026-03-20T11:46:17.292 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.292 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.292 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.292 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=27, curl_handle=0x556b3cce44b0 2026-03-20T11:46:17.292 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f339633c900 20 run: stack=0x556b3cce7430 is io blocked 2026-03-20T11:46:17.292 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.292+0000 7f32f1ffb640 20 link_request req_data=0x556b3cce4bc0 req_data->id=27, curl_handle=0x556b3cce44b0 2026-03-20T11:46:17.292 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.293+0000 7f76b2c57640 -1 req 7007943243117716608 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.320 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceee30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.320 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceee30:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.320 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cd20340:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.320 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f339633c900 15 stack 0x556b3ccee1e0 end 2026-03-20T11:46:17.320 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f339633c900 20 run: stack=0x556b3ccee1e0 is done 2026-03-20T11:46:17.320 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.320 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f339633c900 20 collect(): s=0x556b3cd26450 stack=0x556b3ccee1e0 is complete 2026-03-20T11:46:17.320 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.320 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.320 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.320 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.320 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f339633c900 15 stack 0x556b3cd26450 end 2026-03-20T11:46:17.320 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.320 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f339633c900 20 run: stack=0x556b3cd26450 is done 2026-03-20T11:46:17.320 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.320 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.320 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.320 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.320 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f339633c900 4 starting trim on bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:46:17.320 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3c8879c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.320 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3c8879c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.321 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f339633c900 20 enqueued request req=0x556b3c887640 2026-03-20T11:46:17.321 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.321 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f339633c900 20 req: 0x556b3c887640 2026-03-20T11:46:17.321 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f3359ffb640 20 dequeued request req=0x556b3c887640 2026-03-20T11:46:17.321 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f3359ffb640 20 RGWWQ: empty 2026-03-20T11:46:17.321 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f3359ffb640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 : miss 2026-03-20T11:46:17.321 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f3359ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.321 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f339633c900 20 run: stack=0x556b3cd26450 is io blocked 2026-03-20T11:46:17.321 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f3359ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:46:17.321 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f3359ffb640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 info.flags=0x17 2026-03-20T11:46:17.321 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f3359ffb640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 to cache LRU end 2026-03-20T11:46:17.321 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f3359ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.321 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f3359ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.321 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f3359ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:46:17.321 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f3359ffb640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-1 : miss 2026-03-20T11:46:17.321 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.322+0000 7f3359ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.321 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f3359ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.321 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f3359ffb640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-1 info.flags=0x0 2026-03-20T11:46:17.321 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f3359ffb640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-1 to cache LRU end 2026-03-20T11:46:17.321 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f3359ffb640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-1 : miss 2026-03-20T11:46:17.321 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f3359ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f3359ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f3359ffb640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-1 info.flags=0x0 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f3359ffb640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-1 to cache LRU end 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f3359ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1): adding source pipe: {s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f3359ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1): adding dest pipe: {s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f3359ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f3359ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f3359ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3c8879c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3c8879c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3c8879c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3c8879c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cd20340:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 20 run: stack=0x556b3cd26450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceee30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceee30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 26af447972bca8fb43040b300f46c60d67c7f93de3af2c16d011b2c93e8a1519 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:26af447972bca8fb43040b300f46c60d67c7f93de3af2c16d011b2c93e8a1519 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 91cd9170f4f860f1584f41458a8c3c36c6864a40302d59e38f0506f43a014d41 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=91cd9170f4f860f1584f41458a8c3c36c6864a40302d59e38f0506f43a014d41 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=28, curl_handle=0x556b3cce8410 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f339633c900 20 run: stack=0x556b3ccee1e0 is io blocked 2026-03-20T11:46:17.322 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.323+0000 7f32f1ffb640 20 link_request req_data=0x556b3ccecae0 req_data->id=28, curl_handle=0x556b3cce8410 2026-03-20T11:46:17.323 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.325+0000 7f771451a640 -1 req 7568367970924658734 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd240e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 15 stack 0x556b3ccf26b0 end 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 run: stack=0x556b3ccf26b0 is done 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd23290:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd23290:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 collect(): s=0x556b3ccd07e0 stack=0x556b3ccf26b0 is complete 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 15 stack 0x556b3ccd07e0 end 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is done 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3c88ab90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 15 stack 0x556b3cd2ffa0 end 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 run: stack=0x556b3cd2ffa0 is done 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 collect(): s=0x556b3cd2fd80 stack=0x556b3cd2ffa0 is complete 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 15 stack 0x556b3cd2fd80 end 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is done 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 4 starting trim on bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:46:17.331 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.331 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.331 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.331 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.331 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.331 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 4 starting trim on bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:46:17.331 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.331 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 enqueued request req=0x556b3ccf22b0 2026-03-20T11:46:17.331 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.331 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 req: 0x556b3ccf22b0 2026-03-20T11:46:17.331 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is io blocked 2026-03-20T11:46:17.331 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.331 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.331 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 enqueued request req=0x556b3ccd84d0 2026-03-20T11:46:17.331 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.331 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 req: 0x556b3ccf22b0 2026-03-20T11:46:17.331 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 req: 0x556b3ccd84d0 2026-03-20T11:46:17.331 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f334f7fe640 20 dequeued request req=0x556b3ccf22b0 2026-03-20T11:46:17.331 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f334f7fe640 20 RGWWQ: 2026-03-20T11:46:17.331 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f334f7fe640 20 req: 0x556b3ccd84d0 2026-03-20T11:46:17.331 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f334f7fe640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 : miss 2026-03-20T11:46:17.331 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f334f7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.331 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is io blocked 2026-03-20T11:46:17.331 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f3341fe3640 20 dequeued request req=0x556b3ccd84d0 2026-03-20T11:46:17.331 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f3341fe3640 20 RGWWQ: empty 2026-03-20T11:46:17.331 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f3341fe3640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 : miss 2026-03-20T11:46:17.331 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f3341fe3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f334f7fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:46:17.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f334f7fe640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 info.flags=0x17 2026-03-20T11:46:17.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f334f7fe640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 to cache LRU end 2026-03-20T11:46:17.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f334f7fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f334f7fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f334f7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:46:17.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f334f7fe640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-7 : miss 2026-03-20T11:46:17.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f334f7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f3341fe3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:17.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f3341fe3640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 info.flags=0x17 2026-03-20T11:46:17.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f3341fe3640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 to cache LRU end 2026-03-20T11:46:17.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f3341fe3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f3341fe3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f3341fe3640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:46:17.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f3341fe3640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:46:17.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f3341fe3640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-48 : miss 2026-03-20T11:46:17.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f3341fe3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f334f7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f334f7fe640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-7 info.flags=0x0 2026-03-20T11:46:17.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f334f7fe640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-7 to cache LRU end 2026-03-20T11:46:17.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f334f7fe640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-7 : miss 2026-03-20T11:46:17.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.332+0000 7f334f7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f334f7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f334f7fe640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-7 info.flags=0x0 2026-03-20T11:46:17.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f334f7fe640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-7 to cache LRU end 2026-03-20T11:46:17.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f334f7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12): adding source pipe: {s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f334f7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12): adding dest pipe: {s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f334f7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f334f7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f334f7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f3341fe3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f3341fe3640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-48 info.flags=0x0 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f3341fe3640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-48 to cache LRU end 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f3341fe3640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-48 : miss 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f3341fe3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd23290:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd240e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd240e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = d2e260f924532052863c3ae81963a7e8164862a34a697654a1ad1fb9b4456aeb 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:d2e260f924532052863c3ae81963a7e8164862a34a697654a1ad1fb9b4456aeb 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 87b5876befd745b91a83f2421dd4ce6825a0674eb10e92de5c439e7fc0b4f2cf 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=87b5876befd745b91a83f2421dd4ce6825a0674eb10e92de5c439e7fc0b4f2cf 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=29, curl_handle=0x556b3cc93bb0 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 run: stack=0x556b3cd2ffa0 is io blocked 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f32f1ffb640 20 link_request req_data=0x7f3334005bb0 req_data->id=29, curl_handle=0x556b3cc93bb0 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f3341fe3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f3341fe3640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-48 info.flags=0x0 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f3341fe3640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-48 to cache LRU end 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f3341fe3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8): adding source pipe: {s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f3341fe3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8): adding dest pipe: {s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.333 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f3341fe3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f3341fe3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f3341fe3640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3cce2f00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cced030:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cce2f00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cce2f00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 04920fa8a48e47aee1fe47522f90747adca508e1bc2324e8f0257dd93924b323 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:04920fa8a48e47aee1fe47522f90747adca508e1bc2324e8f0257dd93924b323 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 779abd1ec95a6da16efb4e2bd4feb98b71bf74bb36dbb7431d6de8eb45246da4 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=779abd1ec95a6da16efb4e2bd4feb98b71bf74bb36dbb7431d6de8eb45246da4 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=30, curl_handle=0x556b3c889a20 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f339633c900 20 run: stack=0x556b3ccf26b0 is io blocked 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.333+0000 7f32f1ffb640 20 link_request req_data=0x556b3c88a5f0 req_data->id=30, curl_handle=0x556b3c889a20 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cd20ad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cd20ad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3c8881f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f339633c900 15 stack 0x556b3cce7430 end 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f339633c900 20 run: stack=0x556b3cce7430 is done 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.334 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f339633c900 20 collect(): s=0x556b3cbe9e50 stack=0x556b3cce7430 is complete 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f339633c900 15 stack 0x556b3cbe9e50 end 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is done 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f339633c900 4 starting trim on bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f339633c900 20 enqueued request req=0x556b3cbe9c60 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f339633c900 20 req: 0x556b3cbe9c60 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f3358ff9640 20 dequeued request req=0x556b3cbe9c60 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f3358ff9640 20 RGWWQ: empty 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f3358ff9640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 : miss 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f3358ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is io blocked 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f3358ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f3358ff9640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 info.flags=0x17 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f3358ff9640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 to cache LRU end 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f3358ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f3358ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f3358ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f3358ff9640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-59 : miss 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.334+0000 7f3358ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.335 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.334+0000 7f7706cff640 -1 req 12651900403041176899 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f3358ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.335 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f3358ff9640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-59 info.flags=0x0 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f3358ff9640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-59 to cache LRU end 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f3358ff9640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-59 : miss 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f3358ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.336 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.335+0000 7f7718522640 -1 req 600223125334585686 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f3358ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f3358ff9640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-59 info.flags=0x0 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f3358ff9640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-59 to cache LRU end 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f3358ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19): adding source pipe: {s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f3358ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19): adding dest pipe: {s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f3358ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f3358ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f3358ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cd20ad0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cce6d40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cce6d40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.336 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.337 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.337 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 771aab1246cd62677a83a1a918a810b99be6f37d6117f40d85a48eb2958e52a5 2026-03-20T11:46:17.337 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.337 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.337 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.337 INFO:teuthology.orchestra.run.vm07.stderr:771aab1246cd62677a83a1a918a810b99be6f37d6117f40d85a48eb2958e52a5 2026-03-20T11:46:17.337 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.337 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.337 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.337 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.337 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 3d3628dbbba08b8fed44fe32c95789a996539f4159bd849a57b5b0145a0d7964 2026-03-20T11:46:17.337 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=3d3628dbbba08b8fed44fe32c95789a996539f4159bd849a57b5b0145a0d7964 2026-03-20T11:46:17.337 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.337 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.337 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.337 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=31, curl_handle=0x556b3cce44b0 2026-03-20T11:46:17.337 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f339633c900 20 run: stack=0x556b3cce7430 is io blocked 2026-03-20T11:46:17.337 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.335+0000 7f32f1ffb640 20 link_request req_data=0x556b3cce4bc0 req_data->id=31, curl_handle=0x556b3cce44b0 2026-03-20T11:46:17.337 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.336+0000 7f772d54c640 -1 req 9172400473644694345 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceee30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceee30:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cd20340:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f339633c900 15 stack 0x556b3ccee1e0 end 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f339633c900 20 run: stack=0x556b3ccee1e0 is done 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f339633c900 20 collect(): s=0x556b3cd26450 stack=0x556b3ccee1e0 is complete 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f339633c900 15 stack 0x556b3cd26450 end 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f339633c900 20 run: stack=0x556b3cd26450 is done 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f339633c900 4 starting trim on bucket=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3c8879c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3c8879c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f339633c900 20 enqueued request req=0x556b3c887640 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f339633c900 20 req: 0x556b3c887640 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f33597fa640 20 dequeued request req=0x556b3c887640 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f33597fa640 20 RGWWQ: empty 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f33597fa640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 : miss 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f33597fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f339633c900 20 run: stack=0x556b3cd26450 is io blocked 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f33597fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f33597fa640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 info.flags=0x17 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f33597fa640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 to cache LRU end 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f33597fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f33597fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f33597fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f33597fa640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-6 : miss 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f33597fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f33597fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f33597fa640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-6 info.flags=0x0 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f33597fa640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-6 to cache LRU end 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f33597fa640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-6 : miss 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.366+0000 7f33597fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f33597fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f33597fa640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-6 info.flags=0x0 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f33597fa640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-6 to cache LRU end 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f33597fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11): adding source pipe: {s={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f33597fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11): adding dest pipe: {s={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f33597fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f33597fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f33597fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3c8879c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3c8879c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3c8879c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3c8879c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceee30:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 20 run: stack=0x556b3cd26450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3c8879c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3c8879c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.365 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = ed34162732c7d690c2ee885ad1d4416af7825ea968e84dcbee11a2de63746113 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:ed34162732c7d690c2ee885ad1d4416af7825ea968e84dcbee11a2de63746113 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = bbd7efcfb237178b55c787694dea2111ca664ff963d0548b517c6d56da819251 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=bbd7efcfb237178b55c787694dea2111ca664ff963d0548b517c6d56da819251 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=32, curl_handle=0x556b3cce8410 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f339633c900 20 run: stack=0x556b3ccee1e0 is io blocked 2026-03-20T11:46:17.366 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.367+0000 7f32f1ffb640 20 link_request req_data=0x556b3ccecae0 req_data->id=32, curl_handle=0x556b3cce8410 2026-03-20T11:46:17.366 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.368+0000 7f772c54a640 -1 req 10163690135990942848 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cce2f00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cce2f00:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cced030:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f339633c900 15 stack 0x556b3ccf26b0 end 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f339633c900 20 run: stack=0x556b3ccf26b0 is done 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f339633c900 20 collect(): s=0x556b3cd2fd80 stack=0x556b3ccf26b0 is complete 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f339633c900 15 stack 0x556b3cd2fd80 end 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is done 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f339633c900 4 starting trim on bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3c887ff0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3c887ff0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f339633c900 20 enqueued request req=0x556b3ccd84d0 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f339633c900 20 req: 0x556b3ccd84d0 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f334ffff640 20 dequeued request req=0x556b3ccd84d0 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f334ffff640 20 RGWWQ: empty 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f334ffff640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 : miss 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f334ffff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is io blocked 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f334ffff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f334ffff640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 info.flags=0x17 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f334ffff640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 to cache LRU end 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f334ffff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f334ffff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f334ffff640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f334ffff640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-21 : miss 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f334ffff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f334ffff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f334ffff640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-21 info.flags=0x0 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f334ffff640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-21 to cache LRU end 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f334ffff640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-21 : miss 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f334ffff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f334ffff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f334ffff640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-21 info.flags=0x0 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f334ffff640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-21 to cache LRU end 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f334ffff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26): adding source pipe: {s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.375+0000 7f334ffff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26): adding dest pipe: {s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f334ffff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f334ffff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f334ffff640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3c887ff0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3c887ff0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd240e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd240e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3c887ff0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd23290:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 15 stack 0x556b3cd2ffa0 end 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 run: stack=0x556b3cd2ffa0 is done 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3c887ff0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 collect(): s=0x556b3ccd07e0 stack=0x556b3cd2ffa0 is complete 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 15 stack 0x556b3ccd07e0 end 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is done 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd23290:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 4 starting trim on bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3c887ff0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 1dd8278141a9011065479ab738915432fb736b5ed7d687462d3f39025dff3ad5 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:1dd8278141a9011065479ab738915432fb736b5ed7d687462d3f39025dff3ad5 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 5601086a9906dbcc51db29348b5804b2e6a9e7d5b34e55b2698cf33c2908a492 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5601086a9906dbcc51db29348b5804b2e6a9e7d5b34e55b2698cf33c2908a492 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=33, curl_handle=0x556b3cc93bb0 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 run: stack=0x556b3cd2ffa0 is io blocked 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3c887ff0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 enqueued request req=0x556b3ccf22b0 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 req: 0x556b3ccf22b0 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is io blocked 2026-03-20T11:46:17.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f32f1ffb640 20 link_request req_data=0x7f3334005bb0 req_data->id=33, curl_handle=0x556b3cc93bb0 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f334effd640 20 dequeued request req=0x556b3ccf22b0 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f334effd640 20 RGWWQ: empty 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f334effd640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 : miss 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f334effd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f334effd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f334effd640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 info.flags=0x17 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f334effd640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 to cache LRU end 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f334effd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f334effd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f334effd640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f334effd640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-41 : miss 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f334effd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f334effd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f334effd640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-41 info.flags=0x0 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f334effd640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-41 to cache LRU end 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f334effd640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-41 : miss 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.376+0000 7f334effd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f334effd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f334effd640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-41 info.flags=0x0 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f334effd640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-41 to cache LRU end 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f334effd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1): adding source pipe: {s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f334effd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1): adding dest pipe: {s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f334effd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f334effd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f334effd640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3c887ff0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3c887ff0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3c887ff0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3c887ff0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cce2f00:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd240e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd240e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = c29f8611dacc59d52bc0c75f166f25673637065fa62001ce54feb031e0b9dc98 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:c29f8611dacc59d52bc0c75f166f25673637065fa62001ce54feb031e0b9dc98 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 2e741342ab553d11129f5adff3aa873fe6bb687896e3a6ad8c8aae3376d1b0a9 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2e741342ab553d11129f5adff3aa873fe6bb687896e3a6ad8c8aae3376d1b0a9 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=34, curl_handle=0x556b3c889a20 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f339633c900 20 run: stack=0x556b3ccf26b0 is io blocked 2026-03-20T11:46:17.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.377+0000 7f32f1ffb640 20 link_request req_data=0x556b3c88a5f0 req_data->id=34, curl_handle=0x556b3c889a20 2026-03-20T11:46:17.377 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.377+0000 7f765c3aa640 -1 req 11725738912509737662 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cce6d40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cce6d40:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cd20ad0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f339633c900 15 stack 0x556b3cce7430 end 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f339633c900 20 run: stack=0x556b3cce7430 is done 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f339633c900 20 collect(): s=0x556b3cbe9e50 stack=0x556b3cce7430 is complete 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f339633c900 15 stack 0x556b3cbe9e50 end 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is done 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f339633c900 4 starting trim on bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f339633c900 20 enqueued request req=0x556b3cbe9c60 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f339633c900 20 req: 0x556b3cbe9c60 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is io blocked 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f334dffb640 20 dequeued request req=0x556b3cbe9c60 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f334dffb640 20 RGWWQ: empty 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f334dffb640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 : miss 2026-03-20T11:46:17.378 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f334dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.378 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.378+0000 7f76633b8640 -1 req 2629945292661905041 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f334dffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f334dffb640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 info.flags=0x17 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f334dffb640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 to cache LRU end 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f334dffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f334dffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f334dffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f334dffb640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-49 : miss 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.378+0000 7f334dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f334dffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f334dffb640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-49 info.flags=0x0 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f334dffb640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-49 to cache LRU end 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f334dffb640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-49 : miss 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f334dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f334dffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f334dffb640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-49 info.flags=0x0 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f334dffb640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-49 to cache LRU end 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f334dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9): adding source pipe: {s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f334dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9): adding dest pipe: {s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f334dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f334dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f334dffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cd20ad0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cce6d40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cce6d40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.379 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.380 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.380 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = edeb22f053aaa314ab55fcf17d29339f53ac38f3415b7dfadaedb72c3e11ce6a 2026-03-20T11:46:17.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.380 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.380 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.380 INFO:teuthology.orchestra.run.vm07.stderr:edeb22f053aaa314ab55fcf17d29339f53ac38f3415b7dfadaedb72c3e11ce6a 2026-03-20T11:46:17.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = c8c23ceb58a7831cc1c0b98fcaa53bc69b8f906604e3ebc2a4e1c2058c378211 2026-03-20T11:46:17.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c8c23ceb58a7831cc1c0b98fcaa53bc69b8f906604e3ebc2a4e1c2058c378211 2026-03-20T11:46:17.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=35, curl_handle=0x556b3cce44b0 2026-03-20T11:46:17.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f339633c900 20 run: stack=0x556b3cce7430 is io blocked 2026-03-20T11:46:17.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.379+0000 7f32f1ffb640 20 link_request req_data=0x556b3cce4bc0 req_data->id=35, curl_handle=0x556b3cce44b0 2026-03-20T11:46:17.380 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.380+0000 7f76673c0640 -1 req 10142583032647271740 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.407 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3c8879c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.407 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3c8879c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.407 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceee30:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.407 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f339633c900 15 stack 0x556b3ccee1e0 end 2026-03-20T11:46:17.407 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f339633c900 20 run: stack=0x556b3ccee1e0 is done 2026-03-20T11:46:17.407 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.407 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f339633c900 20 collect(): s=0x556b3cd26450 stack=0x556b3ccee1e0 is complete 2026-03-20T11:46:17.407 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.407 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.407 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.407 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.407 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f339633c900 15 stack 0x556b3cd26450 end 2026-03-20T11:46:17.407 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.407 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f339633c900 20 run: stack=0x556b3cd26450 is done 2026-03-20T11:46:17.407 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.407 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.407 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f339633c900 4 starting trim on bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd31330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd31330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f339633c900 20 enqueued request req=0x556b3c887640 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f339633c900 20 req: 0x556b3c887640 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f334e7fc640 20 dequeued request req=0x556b3c887640 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f334e7fc640 20 RGWWQ: empty 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f334e7fc640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 : miss 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f334e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f339633c900 20 run: stack=0x556b3cd26450 is io blocked 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f334e7fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f334e7fc640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 info.flags=0x17 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f334e7fc640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 to cache LRU end 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f334e7fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f334e7fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f334e7fc640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f334e7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f334e7fc640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-45 : miss 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.409+0000 7f334e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f334e7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f334e7fc640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-45 info.flags=0x0 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f334e7fc640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-45 to cache LRU end 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f334e7fc640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-45 : miss 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f334e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f334e7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f334e7fc640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-45 info.flags=0x0 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f334e7fc640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-45 to cache LRU end 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f334e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5): adding source pipe: {s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f334e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5): adding dest pipe: {s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f334e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f334e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f334e7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd31330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd31330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd31330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd31330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3c8879c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 20 run: stack=0x556b3cd26450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceee30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceee30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 0eb8e676e347579dd98c25083e381de86e13527cb3d618c3fb163264d48690ce 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:0eb8e676e347579dd98c25083e381de86e13527cb3d618c3fb163264d48690ce 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 82753f4265ba7ee272ca217064cbc885005eef3719e2454d18fcf4ff071f2db5 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=82753f4265ba7ee272ca217064cbc885005eef3719e2454d18fcf4ff071f2db5 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=36, curl_handle=0x556b3cce8410 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f339633c900 20 run: stack=0x556b3ccee1e0 is io blocked 2026-03-20T11:46:17.409 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.410+0000 7f32f1ffb640 20 link_request req_data=0x556b3ccecae0 req_data->id=36, curl_handle=0x556b3cce8410 2026-03-20T11:46:17.409 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.411+0000 7f772ed4f640 -1 req 15970338305262884433 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd23290:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f339633c900 15 stack 0x556b3cd2ffa0 end 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f339633c900 20 run: stack=0x556b3cd2ffa0 is done 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f339633c900 20 collect(): s=0x556b3cd2fd80 stack=0x556b3cd2ffa0 is complete 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f339633c900 15 stack 0x556b3cd2fd80 end 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is done 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f339633c900 4 starting trim on bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f339633c900 20 enqueued request req=0x556b3ccd84d0 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f339633c900 20 req: 0x556b3ccd84d0 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f334d7fa640 20 dequeued request req=0x556b3ccd84d0 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f334d7fa640 20 RGWWQ: empty 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f334d7fa640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 : miss 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f334d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.416 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is io blocked 2026-03-20T11:46:17.417 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f334d7fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.417 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f334d7fa640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 info.flags=0x17 2026-03-20T11:46:17.417 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f334d7fa640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 to cache LRU end 2026-03-20T11:46:17.417 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f334d7fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.417 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f334d7fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.417 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f334d7fa640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:46:17.417 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f334d7fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:46:17.417 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f334d7fa640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-33 : miss 2026-03-20T11:46:17.417 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f334d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.417 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f334d7fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.417 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f334d7fa640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-33 info.flags=0x0 2026-03-20T11:46:17.417 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f334d7fa640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-33 to cache LRU end 2026-03-20T11:46:17.417 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f334d7fa640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-33 : miss 2026-03-20T11:46:17.417 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.418+0000 7f334d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd240e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd240e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cce2f00:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 15 stack 0x556b3ccf26b0 end 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 run: stack=0x556b3ccf26b0 is done 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 collect(): s=0x556b3ccd07e0 stack=0x556b3ccf26b0 is complete 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 15 stack 0x556b3ccd07e0 end 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is done 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 4 starting trim on bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce6510:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce6510:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 enqueued request req=0x556b3ccf22b0 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 req: 0x556b3ccf22b0 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is io blocked 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f334cff9640 20 dequeued request req=0x556b3ccf22b0 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f334cff9640 20 RGWWQ: empty 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f334cff9640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 : miss 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f334cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f334d7fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f334d7fa640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-33 info.flags=0x0 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f334d7fa640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-33 to cache LRU end 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f334d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38): adding source pipe: {s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f334d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38): adding dest pipe: {s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f334d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f334d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f334d7fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cce2f00:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd23290:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd23290:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.418 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 60f1a521c05812677e0b5812fb8df22060d384d20b82f84474eee9e1210a5af3 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:60f1a521c05812677e0b5812fb8df22060d384d20b82f84474eee9e1210a5af3 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = a845d44c33a8050fb47249c9e0c1083059aef5184d7d0c4cec8253eaf4937b55 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a845d44c33a8050fb47249c9e0c1083059aef5184d7d0c4cec8253eaf4937b55 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=37, curl_handle=0x556b3c889a20 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f339633c900 20 run: stack=0x556b3ccf26b0 is io blocked 2026-03-20T11:46:17.419 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f32f1ffb640 20 link_request req_data=0x556b3c88a5f0 req_data->id=37, curl_handle=0x556b3c889a20 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f334cff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f334cff9640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 info.flags=0x17 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f334cff9640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 to cache LRU end 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f334cff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f334cff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f334cff9640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f334cff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f334cff9640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-47 : miss 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.419+0000 7f334cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f334cff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f334cff9640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-47 info.flags=0x0 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f334cff9640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-47 to cache LRU end 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f334cff9640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-47 : miss 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f334cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f334cff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f334cff9640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-47 info.flags=0x0 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f334cff9640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-47 to cache LRU end 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f334cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7): adding source pipe: {s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f334cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7): adding dest pipe: {s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f334cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f334cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f334cff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce6510:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce6510:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce6510:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cce6510:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cce6510:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.420 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 748c77127536342af6d2e751ef3653adea25b14806fa8e05481e50979f379827 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:748c77127536342af6d2e751ef3653adea25b14806fa8e05481e50979f379827 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 6b194af80a6540e428f9b9361ceaaf11e108338e156fd6ee3d02a038474b46aa 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=6b194af80a6540e428f9b9361ceaaf11e108338e156fd6ee3d02a038474b46aa 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=38, curl_handle=0x556b3cc93bb0 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f339633c900 20 run: stack=0x556b3cd2ffa0 is io blocked 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.420+0000 7f32f1ffb640 20 link_request req_data=0x7f3334005bb0 req_data->id=38, curl_handle=0x556b3cc93bb0 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cce6d40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cce6d40:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cd20ad0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f339633c900 15 stack 0x556b3cce7430 end 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f339633c900 20 run: stack=0x556b3cce7430 is done 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f339633c900 20 collect(): s=0x556b3cbe9e50 stack=0x556b3cce7430 is complete 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f339633c900 15 stack 0x556b3cbe9e50 end 2026-03-20T11:46:17.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is done 2026-03-20T11:46:17.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f339633c900 4 starting trim on bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:17.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f339633c900 20 enqueued request req=0x556b3cbe9c60 2026-03-20T11:46:17.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f339633c900 20 req: 0x556b3cbe9c60 2026-03-20T11:46:17.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is io blocked 2026-03-20T11:46:17.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f334c7f8640 20 dequeued request req=0x556b3cbe9c60 2026-03-20T11:46:17.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f334c7f8640 20 RGWWQ: empty 2026-03-20T11:46:17.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f334c7f8640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 : miss 2026-03-20T11:46:17.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f334c7f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.422 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.421+0000 7f7723538640 -1 req 5184631691791391858 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f334c7f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f334c7f8640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 info.flags=0x17 2026-03-20T11:46:17.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f334c7f8640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 to cache LRU end 2026-03-20T11:46:17.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f334c7f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f334c7f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f334c7f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:17.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.421+0000 7f334c7f8640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-58 : miss 2026-03-20T11:46:17.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.422+0000 7f334c7f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.422 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.422+0000 7f76cdc8d640 -1 req 12534111167132145845 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.422+0000 7f334c7f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.422+0000 7f334c7f8640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-58 info.flags=0x0 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.422+0000 7f334c7f8640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-58 to cache LRU end 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.422+0000 7f334c7f8640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-58 : miss 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.422+0000 7f334c7f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.422+0000 7f334c7f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.422+0000 7f334c7f8640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-58 info.flags=0x0 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.422+0000 7f334c7f8640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-58 to cache LRU end 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.422+0000 7f334c7f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18): adding source pipe: {s={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f334c7f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18): adding dest pipe: {s={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f334c7f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f334c7f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f334c7f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3ccea780:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cce6d40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3ccea780:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3ccea780:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.423 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.424 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.424 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.424 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.424 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.424 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = d8832c457ccd490ac9f39b5308c803d9ffb48a39bf66488e47b2946d118dbfd0 2026-03-20T11:46:17.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.424 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.424 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.424 INFO:teuthology.orchestra.run.vm07.stderr:d8832c457ccd490ac9f39b5308c803d9ffb48a39bf66488e47b2946d118dbfd0 2026-03-20T11:46:17.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 5f2d0763e2c2ead0c30a88e498b59689782911d89f66b741aa1bc3d1379391ae 2026-03-20T11:46:17.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5f2d0763e2c2ead0c30a88e498b59689782911d89f66b741aa1bc3d1379391ae 2026-03-20T11:46:17.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=39, curl_handle=0x556b3cce44b0 2026-03-20T11:46:17.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f339633c900 20 run: stack=0x556b3cce7430 is io blocked 2026-03-20T11:46:17.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.423+0000 7f32f1ffb640 20 link_request req_data=0x556b3cce4bc0 req_data->id=39, curl_handle=0x556b3cce44b0 2026-03-20T11:46:17.424 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.424+0000 7f76f2cd7640 -1 req 10516679106243093606 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceee30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceee30:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3c8879c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f339633c900 15 stack 0x556b3ccee1e0 end 2026-03-20T11:46:17.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f339633c900 20 run: stack=0x556b3ccee1e0 is done 2026-03-20T11:46:17.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f339633c900 20 collect(): s=0x556b3cd26450 stack=0x556b3ccee1e0 is complete 2026-03-20T11:46:17.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f339633c900 15 stack 0x556b3cd26450 end 2026-03-20T11:46:17.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f339633c900 20 run: stack=0x556b3cd26450 is done 2026-03-20T11:46:17.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.450 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f339633c900 4 starting trim on bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:46:17.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd31330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd31330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f339633c900 20 enqueued request req=0x556b3c887640 2026-03-20T11:46:17.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f339633c900 20 req: 0x556b3c887640 2026-03-20T11:46:17.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f334b7f6640 20 dequeued request req=0x556b3c887640 2026-03-20T11:46:17.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f334b7f6640 20 RGWWQ: empty 2026-03-20T11:46:17.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f334b7f6640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 : miss 2026-03-20T11:46:17.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f334b7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f339633c900 20 run: stack=0x556b3cd26450 is io blocked 2026-03-20T11:46:17.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f334b7f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f334b7f6640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 info.flags=0x17 2026-03-20T11:46:17.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f334b7f6640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 to cache LRU end 2026-03-20T11:46:17.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f334b7f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f334b7f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:17.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f334b7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:46:17.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f334b7f6640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-30 : miss 2026-03-20T11:46:17.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.452+0000 7f334b7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f334b7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f334b7f6640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-30 info.flags=0x0 2026-03-20T11:46:17.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f334b7f6640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-30 to cache LRU end 2026-03-20T11:46:17.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f334b7f6640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-30 : miss 2026-03-20T11:46:17.451 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f334b7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f334b7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f334b7f6640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-30 info.flags=0x0 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f334b7f6640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-30 to cache LRU end 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f334b7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35): adding source pipe: {s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f334b7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35): adding dest pipe: {s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f334b7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f334b7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f334b7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd31330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd31330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd31330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd31330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3c8879c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 20 run: stack=0x556b3cd26450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceee30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceee30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 5eb6a7fbfa74352a74c6c0f8a2959e4a9a622e71a86c25e2242ec44a8219cd25 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:5eb6a7fbfa74352a74c6c0f8a2959e4a9a622e71a86c25e2242ec44a8219cd25 2026-03-20T11:46:17.452 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.453 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.453 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.453 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.453 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = da95adedec08eca229e86c5e3213fde2ff0c8bfb54ba7677197825dae4a7b959 2026-03-20T11:46:17.453 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=da95adedec08eca229e86c5e3213fde2ff0c8bfb54ba7677197825dae4a7b959 2026-03-20T11:46:17.453 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.453 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.453 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.453 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=40, curl_handle=0x556b3cce8410 2026-03-20T11:46:17.453 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f339633c900 20 run: stack=0x556b3ccee1e0 is io blocked 2026-03-20T11:46:17.453 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.453+0000 7f32f1ffb640 20 link_request req_data=0x556b3ccecae0 req_data->id=40, curl_handle=0x556b3cce8410 2026-03-20T11:46:17.453 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.455+0000 7f76f1cd5640 -1 req 17820248822971702801 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd23290:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd23290:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cce2f00:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f339633c900 15 stack 0x556b3ccf26b0 end 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f339633c900 20 run: stack=0x556b3ccf26b0 is done 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f339633c900 20 collect(): s=0x556b3cd2fd80 stack=0x556b3ccf26b0 is complete 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f339633c900 15 stack 0x556b3cd2fd80 end 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is done 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f339633c900 4 starting trim on bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f339633c900 20 enqueued request req=0x556b3ccd84d0 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f339633c900 20 req: 0x556b3ccd84d0 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f334aff5640 20 dequeued request req=0x556b3ccd84d0 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f334aff5640 20 RGWWQ: empty 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f334aff5640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 : miss 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f334aff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is io blocked 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f334aff5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f334aff5640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 info.flags=0x17 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f334aff5640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 to cache LRU end 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f334aff5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f334aff5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f334aff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f334aff5640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-20 : miss 2026-03-20T11:46:17.460 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.462+0000 7f334aff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f334aff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f334aff5640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-20 info.flags=0x0 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f334aff5640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-20 to cache LRU end 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f334aff5640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-20 : miss 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f334aff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cce6510:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 15 stack 0x556b3cd2ffa0 end 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 run: stack=0x556b3cd2ffa0 is done 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 collect(): s=0x556b3ccd07e0 stack=0x556b3cd2ffa0 is complete 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 15 stack 0x556b3ccd07e0 end 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is done 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 4 starting trim on bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cd31b60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cd31b60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 enqueued request req=0x556b3ccf22b0 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 req: 0x556b3ccf22b0 2026-03-20T11:46:17.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is io blocked 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f334bff7640 20 dequeued request req=0x556b3ccf22b0 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f334bff7640 20 RGWWQ: empty 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f334bff7640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 : miss 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f334bff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f334aff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f334aff5640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-20 info.flags=0x0 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f334aff5640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-20 to cache LRU end 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f334aff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25): adding source pipe: {s={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f334aff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25): adding dest pipe: {s={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f334aff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f334aff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f334aff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cced030:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cce2f00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cce2f00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = b82f42439451a3f23f3d032be0029767f24611431aebac750b0bdb84ad9c3ffd 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:b82f42439451a3f23f3d032be0029767f24611431aebac750b0bdb84ad9c3ffd 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = a6bcb242d41ff233896b2185c958d6a5526b54edea6fc88c69772c15ba70b5b5 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a6bcb242d41ff233896b2185c958d6a5526b54edea6fc88c69772c15ba70b5b5 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=41, curl_handle=0x556b3cc93bb0 2026-03-20T11:46:17.462 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f339633c900 20 run: stack=0x556b3cd2ffa0 is io blocked 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f32f1ffb640 20 link_request req_data=0x7f3334005bb0 req_data->id=41, curl_handle=0x556b3cc93bb0 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.463+0000 7f334bff7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.464+0000 7f334bff7640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 info.flags=0x17 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.464+0000 7f334bff7640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 to cache LRU end 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.464+0000 7f334bff7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.464+0000 7f334bff7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.464+0000 7f334bff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.464+0000 7f334bff7640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-52 : miss 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.464+0000 7f334bff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.464+0000 7f334bff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.464+0000 7f334bff7640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-52 info.flags=0x0 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.464+0000 7f334bff7640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-52 to cache LRU end 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.464+0000 7f334bff7640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-52 : miss 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.464+0000 7f334bff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.464+0000 7f334bff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.464+0000 7f334bff7640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-52 info.flags=0x0 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f334bff7640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-52 to cache LRU end 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f334bff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12): adding source pipe: {s={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f334bff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12): adding dest pipe: {s={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f334bff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f334bff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f334bff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cd31b60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cd31b60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3ccea780:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3ccea780:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cd31b60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cce6d40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 15 stack 0x556b3cce7430 end 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 run: stack=0x556b3cce7430 is done 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cd31b60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.463 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 collect(): s=0x556b3cbe9e50 stack=0x556b3cce7430 is complete 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 15 stack 0x556b3cbe9e50 end 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is done 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cce6d40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 4 starting trim on bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3ccea780:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd31b60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3ccea780:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 3d9c73aacedefa47faa87990e22faf67d9f38559b0d07643497a055c461ae181 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:3d9c73aacedefa47faa87990e22faf67d9f38559b0d07643497a055c461ae181 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 09fde845e3678023a59d1dfc01de8617563e00302f954146448e09871377f06e 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=09fde845e3678023a59d1dfc01de8617563e00302f954146448e09871377f06e 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=42, curl_handle=0x556b3cce44b0 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 run: stack=0x556b3cce7430 is io blocked 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd31b60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 enqueued request req=0x556b3cbe9c60 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 req: 0x556b3cbe9c60 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is io blocked 2026-03-20T11:46:17.464 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f32f1ffb640 20 link_request req_data=0x556b3cce4bc0 req_data->id=42, curl_handle=0x556b3cce44b0 2026-03-20T11:46:17.465 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f334a7f4640 20 dequeued request req=0x556b3cbe9c60 2026-03-20T11:46:17.465 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f334a7f4640 20 RGWWQ: empty 2026-03-20T11:46:17.465 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f334a7f4640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 : miss 2026-03-20T11:46:17.465 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.465+0000 7f334a7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.465 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.465+0000 7f765f3b0640 -1 req 14879620948201984661 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.465 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.466+0000 7f334a7f4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:17.465 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.466+0000 7f334a7f4640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 info.flags=0x17 2026-03-20T11:46:17.465 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.466+0000 7f334a7f4640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 to cache LRU end 2026-03-20T11:46:17.465 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.466+0000 7f334a7f4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.465 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.466+0000 7f334a7f4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.465 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.466+0000 7f334a7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:46:17.465 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.466+0000 7f334a7f4640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-42 : miss 2026-03-20T11:46:17.465 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.466+0000 7f334a7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.465 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.466+0000 7f76cc48a640 -1 req 14860955793393559737 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.465 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.466+0000 7f334a7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.465 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.466+0000 7f334a7f4640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-42 info.flags=0x0 2026-03-20T11:46:17.465 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.466+0000 7f334a7f4640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-42 to cache LRU end 2026-03-20T11:46:17.465 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.466+0000 7f334a7f4640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-42 : miss 2026-03-20T11:46:17.465 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.466+0000 7f334a7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.465 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f334a7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.465 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f334a7f4640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-42 info.flags=0x0 2026-03-20T11:46:17.465 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f334a7f4640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-42 to cache LRU end 2026-03-20T11:46:17.465 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f334a7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2): adding source pipe: {s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.465 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f334a7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2): adding dest pipe: {s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.465 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f334a7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.465 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f334a7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.465 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f334a7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd31b60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd31b60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd31b60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd31b60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd31b60:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cce6510:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cce6510:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = ccbf26b8f374117d3d7349eb03719751de23b1762126e458cbf9154295a02408 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:ccbf26b8f374117d3d7349eb03719751de23b1762126e458cbf9154295a02408 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 8d161ed2720ecb71e2dad14006750f0d890fd49fbe7f2d64e0a7d9a4829978fb 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=8d161ed2720ecb71e2dad14006750f0d890fd49fbe7f2d64e0a7d9a4829978fb 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=43, curl_handle=0x556b3c889a20 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f339633c900 20 run: stack=0x556b3ccf26b0 is io blocked 2026-03-20T11:46:17.466 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.467+0000 7f32f1ffb640 20 link_request req_data=0x556b3c88a5f0 req_data->id=43, curl_handle=0x556b3c889a20 2026-03-20T11:46:17.466 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.468+0000 7f76fdced640 -1 req 16274368635347048772 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceee30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceee30:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3c8879c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f339633c900 15 stack 0x556b3ccee1e0 end 2026-03-20T11:46:17.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f339633c900 20 run: stack=0x556b3ccee1e0 is done 2026-03-20T11:46:17.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f339633c900 20 collect(): s=0x556b3cd26450 stack=0x556b3ccee1e0 is complete 2026-03-20T11:46:17.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f339633c900 15 stack 0x556b3cd26450 end 2026-03-20T11:46:17.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f339633c900 20 run: stack=0x556b3cd26450 is done 2026-03-20T11:46:17.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.494 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f339633c900 4 starting trim on bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:46:17.495 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd31330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.495 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd31330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.495 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f339633c900 20 enqueued request req=0x556b3c887640 2026-03-20T11:46:17.495 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.495 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f339633c900 20 req: 0x556b3c887640 2026-03-20T11:46:17.495 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f339633c900 20 run: stack=0x556b3cd26450 is io blocked 2026-03-20T11:46:17.495 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f3349ff3640 20 dequeued request req=0x556b3c887640 2026-03-20T11:46:17.495 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f3349ff3640 20 RGWWQ: empty 2026-03-20T11:46:17.495 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f3349ff3640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 : miss 2026-03-20T11:46:17.495 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f3349ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.495 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f3349ff3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f3349ff3640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 info.flags=0x17 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f3349ff3640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 to cache LRU end 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f3349ff3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f3349ff3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f3349ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f3349ff3640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-2 : miss 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.496+0000 7f3349ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f3349ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f3349ff3640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-2 info.flags=0x0 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f3349ff3640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-2 to cache LRU end 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f3349ff3640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-2 : miss 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f3349ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f3349ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f3349ff3640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-2 info.flags=0x0 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f3349ff3640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-2 to cache LRU end 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f3349ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2): adding source pipe: {s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f3349ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2): adding dest pipe: {s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f3349ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f3349ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f3349ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd31330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd31330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd31330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd31330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceee30:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 20 run: stack=0x556b3cd26450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3c8879c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3c8879c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.496 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.497 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.497 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.497 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.497 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 4f22c7eebad7722ea9ec991879a1d3268061299cdadfa67dc0edda99ff76d8c8 2026-03-20T11:46:17.497 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.497 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.497 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.497 INFO:teuthology.orchestra.run.vm07.stderr:4f22c7eebad7722ea9ec991879a1d3268061299cdadfa67dc0edda99ff76d8c8 2026-03-20T11:46:17.497 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.497 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.497 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.497 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.497 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = a4f17576a7bda0952468eec7059fc76388cabccfa664eb2d1320adf48abd65d7 2026-03-20T11:46:17.497 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a4f17576a7bda0952468eec7059fc76388cabccfa664eb2d1320adf48abd65d7 2026-03-20T11:46:17.497 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.497 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.497 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.497 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=44, curl_handle=0x556b3cce8410 2026-03-20T11:46:17.497 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f339633c900 20 run: stack=0x556b3ccee1e0 is io blocked 2026-03-20T11:46:17.497 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.497+0000 7f32f1ffb640 20 link_request req_data=0x556b3ccecae0 req_data->id=44, curl_handle=0x556b3cce8410 2026-03-20T11:46:17.497 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.499+0000 7f76fb4e8640 -1 req 15206761619445040181 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.504 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cce2f00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.504 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cce2f00:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.504 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cced030:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.504 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f339633c900 15 stack 0x556b3cd2ffa0 end 2026-03-20T11:46:17.504 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f339633c900 20 run: stack=0x556b3cd2ffa0 is done 2026-03-20T11:46:17.504 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.504 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f339633c900 20 collect(): s=0x556b3cd2fd80 stack=0x556b3cd2ffa0 is complete 2026-03-20T11:46:17.504 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.504 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.504 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.504 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.504 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f339633c900 15 stack 0x556b3cd2fd80 end 2026-03-20T11:46:17.504 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.504 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is done 2026-03-20T11:46:17.504 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.504 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.504 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.504 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.504 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f339633c900 4 starting trim on bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:17.504 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.504 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.504 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f339633c900 20 enqueued request req=0x556b3ccd84d0 2026-03-20T11:46:17.504 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.505 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f339633c900 20 req: 0x556b3ccd84d0 2026-03-20T11:46:17.505 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f33497f2640 20 dequeued request req=0x556b3ccd84d0 2026-03-20T11:46:17.505 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f33497f2640 20 RGWWQ: empty 2026-03-20T11:46:17.505 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f33497f2640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 : miss 2026-03-20T11:46:17.505 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f33497f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.505 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is io blocked 2026-03-20T11:46:17.505 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f33497f2640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.505 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f33497f2640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 info.flags=0x17 2026-03-20T11:46:17.505 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f33497f2640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 to cache LRU end 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f33497f2640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f33497f2640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f33497f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f33497f2640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-57 : miss 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.506+0000 7f33497f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f33497f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f33497f2640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-57 info.flags=0x0 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f33497f2640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-57 to cache LRU end 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f33497f2640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-57 : miss 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f33497f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3ccea780:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3ccea780:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cce6d40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f33497f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f33497f2640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-57 info.flags=0x0 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f33497f2640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-57 to cache LRU end 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f33497f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17): adding source pipe: {s={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f33497f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17): adding dest pipe: {s={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f33497f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f33497f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f33497f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 15 stack 0x556b3cce7430 end 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 run: stack=0x556b3cce7430 is done 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 collect(): s=0x556b3ccd07e0 stack=0x556b3cce7430 is complete 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 15 stack 0x556b3ccd07e0 end 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is done 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 4 starting trim on bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cd322b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.506 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cd322b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 enqueued request req=0x556b3c88a060 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 req: 0x556b3c88a060 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is io blocked 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3ccea780:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f3348ff1640 20 dequeued request req=0x556b3c88a060 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f3348ff1640 20 RGWWQ: empty 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f3348ff1640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 : miss 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f3348ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = ac1486c794918ae176ac8bb0cbd85bc5d911ca501f3e72aa1d4cead4d4e9a812 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:ac1486c794918ae176ac8bb0cbd85bc5d911ca501f3e72aa1d4cead4d4e9a812 2026-03-20T11:46:17.507 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 34d123556e842f778e7f0843143f5e9eaf6de30d83e590ce14de9f55f985d965 2026-03-20T11:46:17.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=34d123556e842f778e7f0843143f5e9eaf6de30d83e590ce14de9f55f985d965 2026-03-20T11:46:17.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=45, curl_handle=0x556b3cce44b0 2026-03-20T11:46:17.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f339633c900 20 run: stack=0x556b3cce7430 is io blocked 2026-03-20T11:46:17.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.507+0000 7f32f1ffb640 20 link_request req_data=0x556b3cce4bc0 req_data->id=45, curl_handle=0x556b3cce44b0 2026-03-20T11:46:17.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.508+0000 7f3348ff1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.508+0000 7f3348ff1640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 info.flags=0x17 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.508+0000 7f3348ff1640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 to cache LRU end 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.508+0000 7f3348ff1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.508+0000 7f3348ff1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.508+0000 7f3348ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.508+0000 7f3348ff1640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-29 : miss 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.508+0000 7f3348ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.508+0000 7f3348ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.508+0000 7f3348ff1640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-29 info.flags=0x0 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.508+0000 7f3348ff1640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-29 to cache LRU end 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.508+0000 7f3348ff1640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-29 : miss 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.508+0000 7f3348ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.508+0000 7f3348ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.508+0000 7f3348ff1640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-29 info.flags=0x0 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.508+0000 7f3348ff1640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-29 to cache LRU end 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f3348ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34): adding source pipe: {s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f3348ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34): adding dest pipe: {s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cce6510:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cce6510:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd31b60:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 15 stack 0x556b3ccf26b0 end 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 run: stack=0x556b3ccf26b0 is done 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 collect(): s=0x556b3cbe9e50 stack=0x556b3ccf26b0 is complete 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 15 stack 0x556b3cbe9e50 end 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is done 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 4 starting trim on bucket=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd216b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd216b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f3348ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 enqueued request req=0x556b3cbe9c60 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 req: 0x556b3cbe9c60 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is io blocked 2026-03-20T11:46:17.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f3348ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f3348ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f3348ff1640 20 dequeued request req=0x556b3cbe9c60 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f3348ff1640 20 RGWWQ: empty 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f3348ff1640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 : miss 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f3348ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cd322b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cd322b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cd322b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cd322b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cce6510:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cce2f00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cce2f00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 2addceee528d8ac8906d732ec6aeed30db3ba0304759a5b8558f5e951a1b93d7 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2addceee528d8ac8906d732ec6aeed30db3ba0304759a5b8558f5e951a1b93d7 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 94f2f0bc12b554c66dfcf5432f83d9068485772372a757ed8bfeb6ffdc02313f 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=94f2f0bc12b554c66dfcf5432f83d9068485772372a757ed8bfeb6ffdc02313f 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=46, curl_handle=0x556b3c889a20 2026-03-20T11:46:17.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f32f1ffb640 20 link_request req_data=0x556b3c88a5f0 req_data->id=46, curl_handle=0x556b3c889a20 2026-03-20T11:46:17.510 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.509+0000 7f764d38c640 -1 req 7619558802521144562 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f339633c900 20 run: stack=0x556b3ccf26b0 is io blocked 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f3348ff1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=408 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f3348ff1640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 info.flags=0x17 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f3348ff1640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 to cache LRU end 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f3348ff1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f3348ff1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=171 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f3348ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f3348ff1640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.testx/tenanted-bucket : miss 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f3348ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f3348ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f3348ff1640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.testx/tenanted-bucket info.flags=0x0 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f3348ff1640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.testx/tenanted-bucket to cache LRU end 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f3348ff1640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.testx/tenanted-bucket : miss 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.509+0000 7f3348ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f3348ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f3348ff1640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.testx/tenanted-bucket info.flags=0x0 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f3348ff1640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.testx/tenanted-bucket to cache LRU end 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f3348ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3): adding source pipe: {s={b=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f3348ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3): adding dest pipe: {s={b=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f3348ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f3348ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f3348ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd216b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd216b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd216b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cd216b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cce6d40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd216b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd216b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.511 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.512 INFO:teuthology.orchestra.run.vm07.stderr:bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.512 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.512 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.512 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.512 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.512 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.512 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.512 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 19bb6194026ad2cc555dda6a52eaf44d6f3d387999feeb6c97bc10361c0a9a22 2026-03-20T11:46:17.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.512 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.512 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.512 INFO:teuthology.orchestra.run.vm07.stderr:19bb6194026ad2cc555dda6a52eaf44d6f3d387999feeb6c97bc10361c0a9a22 2026-03-20T11:46:17.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = c201fae0d9d96bd616298f06604dc2e626f9794e249c1709a26d0212ed734079 2026-03-20T11:46:17.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c201fae0d9d96bd616298f06604dc2e626f9794e249c1709a26d0212ed734079 2026-03-20T11:46:17.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=testx%2Ftenanted-bucket%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=47, curl_handle=0x556b3cc93bb0 2026-03-20T11:46:17.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f339633c900 20 run: stack=0x556b3cd2ffa0 is io blocked 2026-03-20T11:46:17.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.510+0000 7f32f1ffb640 20 link_request req_data=0x7f3334005bb0 req_data->id=47, curl_handle=0x556b3cc93bb0 2026-03-20T11:46:17.512 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.511+0000 7f7658ba3640 -1 req 7856587872240250434 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3c8879c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3c8879c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cceee30:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f339633c900 15 stack 0x556b3ccee1e0 end 2026-03-20T11:46:17.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f339633c900 20 run: stack=0x556b3ccee1e0 is done 2026-03-20T11:46:17.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f339633c900 20 collect(): s=0x556b3cd26450 stack=0x556b3ccee1e0 is complete 2026-03-20T11:46:17.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f339633c900 15 stack 0x556b3cd26450 end 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f339633c900 20 run: stack=0x556b3cd26450 is done 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f339633c900 4 starting trim on bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd31330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd31330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f339633c900 20 enqueued request req=0x556b3c887640 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f339633c900 20 req: 0x556b3c887640 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f3347fef640 20 dequeued request req=0x556b3c887640 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f3347fef640 20 RGWWQ: empty 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f3347fef640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 : miss 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f3347fef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.540+0000 7f339633c900 20 run: stack=0x556b3cd26450 is io blocked 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.541+0000 7f3347fef640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.541+0000 7f3347fef640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 info.flags=0x17 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.541+0000 7f3347fef640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 to cache LRU end 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.541+0000 7f3347fef640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.541+0000 7f3347fef640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.541+0000 7f3347fef640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.541+0000 7f3347fef640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-23 : miss 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.541+0000 7f3347fef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.541+0000 7f3347fef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.541+0000 7f3347fef640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-23 info.flags=0x0 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.541+0000 7f3347fef640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-23 to cache LRU end 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.541+0000 7f3347fef640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-23 : miss 2026-03-20T11:46:17.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.541+0000 7f3347fef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f3347fef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f3347fef640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-23 info.flags=0x0 2026-03-20T11:46:17.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f3347fef640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-23 to cache LRU end 2026-03-20T11:46:17.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f3347fef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28): adding source pipe: {s={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f3347fef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28): adding dest pipe: {s={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f3347fef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f3347fef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f3347fef640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:17.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd31330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd31330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd31330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3cd31330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cd31330:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 20 run: stack=0x556b3cd26450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cd356d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cd356d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 36357a8eefb7d6e54e02a0262b12380ba2078e2f81547780b359b2e1d9a75a35 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:36357a8eefb7d6e54e02a0262b12380ba2078e2f81547780b359b2e1d9a75a35 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 7b6ac842d09d6d56b285fe759d8f359667d93d795cdffcbc17e821c5a5902921 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=7b6ac842d09d6d56b285fe759d8f359667d93d795cdffcbc17e821c5a5902921 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=48, curl_handle=0x556b3cce8410 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f32f1ffb640 20 link_request req_data=0x556b3ccecae0 req_data->id=48, curl_handle=0x556b3cce8410 2026-03-20T11:46:17.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.542+0000 7f339633c900 20 run: stack=0x556b3ccee1e0 is io blocked 2026-03-20T11:46:17.542 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.544+0000 7f763f370640 -1 req 16744367777736325379 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3ccea780:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f339633c900 15 stack 0x556b3cce7430 end 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f339633c900 20 run: stack=0x556b3cce7430 is done 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f339633c900 20 collect(): s=0x556b3cd2fd80 stack=0x556b3cce7430 is complete 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f339633c900 15 stack 0x556b3cd2fd80 end 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is done 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f339633c900 4 starting trim on bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f339633c900 20 enqueued request req=0x556b3ccf22b0 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f339633c900 20 req: 0x556b3ccf22b0 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is io blocked 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f33477ee640 20 dequeued request req=0x556b3ccf22b0 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f33477ee640 20 RGWWQ: empty 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f33477ee640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 : miss 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f33477ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f33477ee640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f33477ee640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 info.flags=0x17 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f33477ee640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 to cache LRU end 2026-03-20T11:46:17.548 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f33477ee640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:17.549 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f33477ee640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:17.549 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f33477ee640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:46:17.549 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f33477ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:46:17.549 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f33477ee640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-34 : miss 2026-03-20T11:46:17.549 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f33477ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.549 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f33477ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.549 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f33477ee640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-34 info.flags=0x0 2026-03-20T11:46:17.549 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f33477ee640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-34 to cache LRU end 2026-03-20T11:46:17.549 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f33477ee640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-34 : miss 2026-03-20T11:46:17.549 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.550+0000 7f33477ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:17.549 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f33477ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:17.549 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f33477ee640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-34 info.flags=0x0 2026-03-20T11:46:17.549 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f33477ee640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-34 to cache LRU end 2026-03-20T11:46:17.549 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f33477ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39): adding source pipe: {s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.549 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f33477ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39): adding dest pipe: {s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.549 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f33477ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:46:17.549 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f33477ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:46:17.549 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f33477ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccf0580:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3ccea780:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:46:17 +0000 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114617Z 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = b80ee36341106a822ded1495b3b7a69cfc8e8c4464ad362291ff2920ca10c8f3 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114617Z 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:b80ee36341106a822ded1495b3b7a69cfc8e8c4464ad362291ff2920ca10c8f3 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = fc590c2b95575b4d8761d12d6a5f8cdaad009772075e39146077f616c743ae64 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=fc590c2b95575b4d8761d12d6a5f8cdaad009772075e39146077f616c743ae64 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114617Z 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 20 register_request mgr=0x7ffc83d343e0 req_data->id=49, curl_handle=0x556b3cce44b0 2026-03-20T11:46:17.550 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f32f1ffb640 20 link_request req_data=0x556b3cce4bc0 req_data->id=49, curl_handle=0x556b3cce44b0 2026-03-20T11:46:17.551 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.551+0000 7f339633c900 20 run: stack=0x556b3cce7430 is io blocked 2026-03-20T11:46:17.551 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.552+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cce2f00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.551 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.552+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cce2f00:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.551 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.552+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cce6510:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.551 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.552+0000 7f339633c900 15 stack 0x556b3ccf26b0 end 2026-03-20T11:46:17.551 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.552+0000 7f339633c900 20 run: stack=0x556b3ccf26b0 is done 2026-03-20T11:46:17.551 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.552+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.551 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.552+0000 7f339633c900 20 collect(): s=0x556b3ccd07e0 stack=0x556b3ccf26b0 is complete 2026-03-20T11:46:17.551 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.552+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.551 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.552+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.551 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.552+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.551 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.552+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3ccd5410:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.551 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.552+0000 7f339633c900 15 stack 0x556b3ccd07e0 end 2026-03-20T11:46:17.551 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.552+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.551 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.552+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is done 2026-03-20T11:46:17.551 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.552+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.551 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.552+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.551 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.552+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.551 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd216b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.551 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd216b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.551 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd216b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.551 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cd216b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.551 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cce6d40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 15 stack 0x556b3cd2ffa0 end 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 run: stack=0x556b3cd2ffa0 is done 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 collect(): s=0x556b3cbe9e50 stack=0x556b3cd2ffa0 is complete 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 10 min_generation is 1 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce6d40:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.0 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.1 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.2 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.3 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.4 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.5 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.6 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.7 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.8 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.9 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3.10 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cce2f00:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cd216b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd21ca0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 cr:s=0x556b3ccd2070:op=0x556b3c878670:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 cr:s=0x556b3cd1c9c0:op=0x556b3c878c60:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 cr:s=0x556b3cd38bc0:op=0x556b3c879250:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 cr:s=0x556b3cd32ff0:op=0x556b3c879840:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 cr:s=0x556b3cd33160:op=0x556b3c879e30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 cr:s=0x556b3cd310a0:op=0x556b3c87a420:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 cr:s=0x556b3cd36270:op=0x556b3c87aa10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 cr:s=0x556b3cd36b30:op=0x556b3c87b000:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cce2f00:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 run: stack=0x556b3cd2ffa0 is io blocked 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cd216b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is io blocked 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd21ca0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 run: stack=0x556b3ccf26b0 is io blocked 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 cr:s=0x556b3ccd2070:op=0x556b3c878670:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 run: stack=0x556b3ccd2070 is io blocked 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 cr:s=0x556b3cd1c9c0:op=0x556b3c878c60:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 run: stack=0x556b3cd1c9c0 is io blocked 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 cr:s=0x556b3cd38bc0:op=0x556b3c879250:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 run: stack=0x556b3cd38bc0 is io blocked 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 cr:s=0x556b3cd32ff0:op=0x556b3c879840:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 run: stack=0x556b3cd32ff0 is io blocked 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 cr:s=0x556b3cd33160:op=0x556b3c879e30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 run: stack=0x556b3cd33160 is io blocked 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 cr:s=0x556b3cd310a0:op=0x556b3c87a420:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.552 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 run: stack=0x556b3cd310a0 is io blocked 2026-03-20T11:46:17.553 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.553+0000 7f339633c900 20 cr:s=0x556b3cd36270:op=0x556b3c87aa10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.553 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.554+0000 7f339633c900 20 run: stack=0x556b3cd36270 is io blocked 2026-03-20T11:46:17.553 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.554+0000 7f339633c900 20 cr:s=0x556b3cd36b30:op=0x556b3c87b000:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.553 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.554+0000 7f339633c900 20 run: stack=0x556b3cd36b30 is io blocked 2026-03-20T11:46:17.553 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:46:17.555+0000 7f7643b79640 -1 req 18445939553467562250 0.004000004s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd21ca0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd21ca0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3cd33160:op=0x556b3c879e30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3cd310a0:op=0x556b3c87a420:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3cd32ff0:op=0x556b3c879840:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3ccd2070:op=0x556b3c878670:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cce2f00:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd21ca0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3cd33160:op=0x556b3c879e30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3cd310a0:op=0x556b3c87a420:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3cd32ff0:op=0x556b3c879840:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3ccd2070:op=0x556b3c878670:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cce2f00:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3ccf26b0:op=0x556b3cd21ca0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 15 stack 0x556b3ccf26b0 end 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 run: stack=0x556b3ccf26b0 is done 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3cd33160:op=0x556b3c879e30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3cd310a0:op=0x556b3c87a420:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3cd32ff0:op=0x556b3c879840:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3ccd2070:op=0x556b3c878670:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cce2f00:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce6d40:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3cd1c9c0:op=0x556b3c878c60:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3cd33160:op=0x556b3c879e30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 15 stack 0x556b3cd33160 end 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 run: stack=0x556b3cd33160 is done 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3cd310a0:op=0x556b3c87a420:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 15 stack 0x556b3cd310a0 end 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 run: stack=0x556b3cd310a0 is done 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3cd32ff0:op=0x556b3c879840:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 15 stack 0x556b3cd32ff0 end 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 run: stack=0x556b3cd32ff0 is done 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3ccd2070:op=0x556b3c878670:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.554 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 15 stack 0x556b3ccd2070 end 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 run: stack=0x556b3ccd2070 is done 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3cd2ffa0:op=0x556b3cce2f00:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 15 stack 0x556b3cd2ffa0 end 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 run: stack=0x556b3cd2ffa0 is done 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3cd1c9c0:op=0x556b3c878c60:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce6d40:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3cd1c9c0:op=0x556b3c878c60:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce6d40:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3cd1c9c0:op=0x556b3c878c60:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 15 stack 0x556b3cd1c9c0 end 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 run: stack=0x556b3cd1c9c0 is done 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce6d40:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce6d40:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce6d40:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce6d40:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.555+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 cr:s=0x556b3cd38bc0:op=0x556b3c879250:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 cr:s=0x556b3cd38bc0:op=0x556b3c879250:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 cr:s=0x556b3cd36270:op=0x556b3c87aa10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 cr:s=0x556b3cd36b30:op=0x556b3c87b000:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 cr:s=0x556b3cd38bc0:op=0x556b3c879250:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 cr:s=0x556b3cd36270:op=0x556b3c87aa10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 cr:s=0x556b3cd36b30:op=0x556b3c87b000:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 cr:s=0x556b3cd38bc0:op=0x556b3c879250:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 15 stack 0x556b3cd38bc0 end 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 run: stack=0x556b3cd38bc0 is done 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 cr:s=0x556b3cd36270:op=0x556b3c87aa10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 cr:s=0x556b3cd36b30:op=0x556b3c87b000:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce6d40:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 cr:s=0x556b3cd36270:op=0x556b3c87aa10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 15 stack 0x556b3cd36270 end 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 run: stack=0x556b3cd36270 is done 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 cr:s=0x556b3cd36b30:op=0x556b3c87b000:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 15 stack 0x556b3cd36b30 end 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 run: stack=0x556b3cd36b30 is done 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce6d40:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce6d40:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cd216b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cd216b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cd216b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 cr:s=0x556b3ccd07e0:op=0x556b3cd216b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 15 stack 0x556b3ccd07e0 end 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 run: stack=0x556b3ccd07e0 is done 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce6d40:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce2f00:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce2f00:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 enqueued request req=0x556b3c8784a0 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 req: 0x556b3c8784a0 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is io blocked 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f33467ec640 20 dequeued request req=0x556b3c8784a0 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.556+0000 7f33467ec640 20 RGWWQ: empty 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.557+0000 7f33467ec640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 info.flags=0x11 2026-03-20T11:46:17.555 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.557+0000 7f33467ec640 10 rgw async rados processor: moving b2.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 to cache LRU end 2026-03-20T11:46:17.556 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.557+0000 7f33467ec640 10 rgw async rados processor: distributing notification oid=b2.rgw.control:notify.5 cni=[op: 0, obj: b2.rgw.meta:root:.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3, ofs0, ns] 2026-03-20T11:46:17.556 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.558+0000 7f336d7fa640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 257698037870 cookie 93919070229952 notifier 4705 bl.length()=599 2026-03-20T11:46:17.556 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.558+0000 7f336d7fa640 10 rgw watcher librados: cache put: name=b2.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 info.flags=0x11 2026-03-20T11:46:17.556 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.558+0000 7f336d7fa640 10 rgw watcher librados: moving b2.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 to cache LRU end 2026-03-20T11:46:17.556 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.558+0000 7f33467ec640 20 rgw async rados processor: handle_bi_update(): bucket=testx:tenanted-bucket[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3]): orig_sources= new_sources= 2026-03-20T11:46:17.556 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.558+0000 7f33467ec640 20 rgw async rados processor: handle_bi_update(): bucket=testx:tenanted-bucket[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3]): potential sources added=[] removed=[] 2026-03-20T11:46:17.556 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.558+0000 7f33467ec640 20 rgw async rados processor: handle_bi_update(): bucket=testx:tenanted-bucket[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3]): orig_dests= new_dests= 2026-03-20T11:46:17.556 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.558+0000 7f33467ec640 20 rgw async rados processor: handle_bi_update(): bucket=testx:tenanted-bucket[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3]): potential dests added=[] removed=[] 2026-03-20T11:46:17.556 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.558+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce2f00:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:17.556 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.558+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce2f00:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:17.556 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.558+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce2f00:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:17.556 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.558+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cce2f00:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:17.556 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.558+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.556 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.558+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3c87d9c0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:17.556 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.558+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3c87d9c0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:17.556 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.558+0000 7f339633c900 20 enqueued request req=0x556b3c813f40 2026-03-20T11:46:17.556 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.558+0000 7f339633c900 20 RGWWQ: 2026-03-20T11:46:17.556 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.558+0000 7f339633c900 20 req: 0x556b3c813f40 2026-03-20T11:46:17.556 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.558+0000 7f3346fed640 20 dequeued request req=0x556b3c813f40 2026-03-20T11:46:17.556 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.558+0000 7f3346fed640 20 RGWWQ: empty 2026-03-20T11:46:17.556 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.558+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is io blocked 2026-03-20T11:46:17.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.559+0000 7f3346fed640 10 rgw async rados processor: removing b2.rgw.meta+root+.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 from cache 2026-03-20T11:46:17.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.559+0000 7f3346fed640 10 rgw async rados processor: distributing notification oid=b2.rgw.control:notify.5 cni=[op: 1, obj: b2.rgw.meta:root:.bucket.meta.testx:tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3, ofs0, ns] 2026-03-20T11:46:17.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.559+0000 7f336dffb640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 257698037871 cookie 93919070229952 notifier 4705 bl.length()=217 2026-03-20T11:46:17.558 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.561+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3c87d9c0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:17.558 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.561+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3c87d9c0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:17.558 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.561+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3c87d9c0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:17.558 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.561+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3c87d9c0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:17.558 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.561+0000 7f339633c900 20 cr:s=0x556b3cbe9e50:op=0x556b3cccf270:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.558 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.561+0000 7f339633c900 20 trim: trimmed bucket instance testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3 2026-03-20T11:46:17.558 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.561+0000 7f339633c900 15 stack 0x556b3cbe9e50 end 2026-03-20T11:46:17.558 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.561+0000 7f339633c900 20 run: stack=0x556b3cbe9e50 is done 2026-03-20T11:46:17.558 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.561+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.558 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.561+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.585+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cd356d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.585+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cd356d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.585+0000 7f339633c900 20 cr:s=0x556b3ccee1e0:op=0x556b3cd31330:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.585+0000 7f339633c900 15 stack 0x556b3ccee1e0 end 2026-03-20T11:46:17.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.585+0000 7f339633c900 20 run: stack=0x556b3ccee1e0 is done 2026-03-20T11:46:17.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.585+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.585+0000 7f339633c900 20 collect(): s=0x556b3cd26450 stack=0x556b3ccee1e0 is complete 2026-03-20T11:46:17.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.585+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.585+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.585+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.585+0000 7f339633c900 20 cr:s=0x556b3cd26450:op=0x556b3ccd4140:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.585+0000 7f339633c900 15 stack 0x556b3cd26450 end 2026-03-20T11:46:17.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.585+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.585+0000 7f339633c900 20 run: stack=0x556b3cd26450 is done 2026-03-20T11:46:17.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.585+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.585+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.583 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.585+0000 7f339633c900 20 run: stack=0x556b3cccded0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:17.594 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.596+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:17.594 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.596+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3cced030:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:17.594 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.596+0000 7f339633c900 20 cr:s=0x556b3cce7430:op=0x556b3ccea780:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:17.594 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.596+0000 7f339633c900 15 stack 0x556b3cce7430 end 2026-03-20T11:46:17.594 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.596+0000 7f339633c900 20 run: stack=0x556b3cce7430 is done 2026-03-20T11:46:17.594 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.596+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:17.594 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.596+0000 7f339633c900 20 collect(): s=0x556b3cd2fd80 stack=0x556b3cce7430 is complete 2026-03-20T11:46:17.594 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.596+0000 7f339633c900 10 min_generation is 0 2026-03-20T11:46:17.594 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.596+0000 7f339633c900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:17.594 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.596+0000 7f339633c900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:17.594 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.596+0000 7f339633c900 20 cr:s=0x556b3cd2fd80:op=0x556b3ccd2e70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:17.594 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.596+0000 7f339633c900 15 stack 0x556b3cd2fd80 end 2026-03-20T11:46:17.594 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.596+0000 7f339633c900 20 stack->operate() returned ret=-22 2026-03-20T11:46:17.594 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.596+0000 7f339633c900 20 run: stack=0x556b3cd2fd80 is done 2026-03-20T11:46:17.595 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.596+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:17.595 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.596+0000 7f339633c900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:17.595 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.596+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3ccd15e0:27BucketTrimInstanceCollectCR: operate() returned r=-22 2026-03-20T11:46:17.595 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.596+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3cccd7d0:12BucketTrimCR: operate() 2026-03-20T11:46:17.595 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.596+0000 7f339633c900 20 writing bucket trim marker=16:ca472e1e:root::.bucket.meta.dipepu-54%3a4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14:head 2026-03-20T11:46:17.595 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.596+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3cd356d0:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:17.595 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.596+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3cd356d0:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:17.595 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.596+0000 7f339633c900 20 run: stack=0x556b3cccded0 is io blocked 2026-03-20T11:46:17.596 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.598+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3cd356d0:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:17.596 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.598+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3cd356d0:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:17.596 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.598+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3cd356d0:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:17.596 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.598+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3cd356d0:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:17.596 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.598+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3cccd7d0:12BucketTrimCR: operate() 2026-03-20T11:46:17.596 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.598+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3cd356d0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:17.596 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.598+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3cd356d0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:17.596 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.598+0000 7f339633c900 20 run: stack=0x556b3cccded0 is io blocked 2026-03-20T11:46:17.596 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.598+0000 7f336dffb640 20 trim: bucket trim completed 2026-03-20T11:46:17.596 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.598+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3cd356d0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:17.596 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.598+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3cd356d0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:17.596 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.598+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3cd356d0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:17.596 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.598+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3cd356d0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:17.597 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.598+0000 7f339633c900 20 cr:s=0x556b3cccded0:op=0x556b3cccd7d0:12BucketTrimCR: operate() 2026-03-20T11:46:17.597 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.598+0000 7f339633c900 4 bucket index log processing completed in 0.752694547s 2026-03-20T11:46:17.597 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.598+0000 7f339633c900 15 stack 0x556b3cccded0 end 2026-03-20T11:46:17.597 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.598+0000 7f339633c900 20 run: stack=0x556b3cccded0 is done 2026-03-20T11:46:17.603 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.605+0000 7f339633c900 20 remove_watcher() i=1 2026-03-20T11:46:17.603 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.605+0000 7f339633c900 2 removed watcher, disabling cache 2026-03-20T11:46:17.603 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.605+0000 7f339633c900 20 remove_watcher() i=4 2026-03-20T11:46:17.603 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.605+0000 7f339633c900 20 remove_watcher() i=5 2026-03-20T11:46:17.603 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.605+0000 7f339633c900 20 remove_watcher() i=7 2026-03-20T11:46:17.603 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.605+0000 7f339633c900 20 remove_watcher() i=2 2026-03-20T11:46:17.604 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.605+0000 7f339633c900 20 remove_watcher() i=0 2026-03-20T11:46:17.604 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.605+0000 7f339633c900 20 remove_watcher() i=6 2026-03-20T11:46:17.604 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:46:17.605+0000 7f339633c900 20 remove_watcher() i=3 2026-03-20T11:46:22.611 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '50', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:46:22.611 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '50', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:46:22.611 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bilog autotrim --rgw-sync-log-trim-max-buckets 50 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 20 --debug-ms 0 2026-03-20T11:46:22.649 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:22.649 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:22.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.667+0000 7f4e96f52900 20 rados->read ofs=0 len=0 2026-03-20T11:46:22.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.668+0000 7f4e96f52900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:46:22.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.668+0000 7f4e96f52900 20 rados->read ofs=0 len=0 2026-03-20T11:46:22.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.669+0000 7f4e96f52900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:46:22.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.669+0000 7f4e96f52900 20 realm test-realm 6a789b89-c2b8-45f5-82cc-fb80234d0c6c 2026-03-20T11:46:22.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.669+0000 7f4e96f52900 20 rados->read ofs=0 len=0 2026-03-20T11:46:22.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.669+0000 7f4e96f52900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:46:22.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.669+0000 7f4e96f52900 20 rados->read ofs=0 len=0 2026-03-20T11:46:22.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.670+0000 7f4e96f52900 20 rados_obj.operate() r=0 bl.length=10 2026-03-20T11:46:22.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.670+0000 7f4e96f52900 20 rados->read ofs=0 len=0 2026-03-20T11:46:22.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.670+0000 7f4e96f52900 20 rados_obj.operate() r=0 bl.length=2020 2026-03-20T11:46:22.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.670+0000 7f4e96f52900 20 rados->read ofs=0 len=0 2026-03-20T11:46:22.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.670+0000 7f4e96f52900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:46:22.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.670+0000 7f4e96f52900 20 rados->read ofs=0 len=0 2026-03-20T11:46:22.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.670+0000 7f4e96f52900 20 rados_obj.operate() r=0 bl.length=1030 2026-03-20T11:46:22.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.671+0000 7f4e96f52900 20 zone a1 found 2026-03-20T11:46:22.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.671+0000 7f4e96f52900 4 Realm: test-realm (6a789b89-c2b8-45f5-82cc-fb80234d0c6c) 2026-03-20T11:46:22.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.671+0000 7f4e96f52900 4 ZoneGroup: a (a) 2026-03-20T11:46:22.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.671+0000 7f4e96f52900 4 Zone: a1 (4b634ebf-58f5-45bc-839a-cb4b425e773a) 2026-03-20T11:46:22.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.671+0000 7f4e96f52900 4 using period configuration: cb2fcdce-2c1b-429a-8601-a284ee3a2381:12 2026-03-20T11:46:22.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.671+0000 7f4e96f52900 20 period zonegroup name a 2026-03-20T11:46:22.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.671+0000 7f4e96f52900 20 using current period zonegroup a 2026-03-20T11:46:22.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.671+0000 7f4e96f52900 20 rados->read ofs=0 len=0 2026-03-20T11:46:22.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.671+0000 7f4e96f52900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:22.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.671+0000 7f4e96f52900 20 rados->read ofs=0 len=0 2026-03-20T11:46:22.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.671+0000 7f4e96f52900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:22.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.671+0000 7f4e96f52900 20 init(): pipe_map (zone=4b634ebf-58f5-45bc-839a-cb4b425e773a bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:46:22.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.671+0000 7f4e96f52900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:22.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.671+0000 7f4e96f52900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:22.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.671+0000 7f4e96f52900 20 rados->read ofs=0 len=0 2026-03-20T11:46:22.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.672+0000 7f4e96f52900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:22.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.672+0000 7f4e96f52900 20 rados->read ofs=0 len=0 2026-03-20T11:46:22.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.672+0000 7f4e96f52900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:22.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.672+0000 7f4e96f52900 20 init(): pipe_map (zone=51055576-6662-409d-8abb-ecaeefd3dd80 bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:46:22.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.672+0000 7f4e96f52900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:22.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.672+0000 7f4e96f52900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:22.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.672+0000 7f4e96f52900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:22.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.672+0000 7f4e96f52900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:22.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.672+0000 7f4e96f52900 20 started sync module instance, tier type = 2026-03-20T11:46:22.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.672+0000 7f4e96f52900 20 generating connection object for zone a2 id 51055576-6662-409d-8abb-ecaeefd3dd80 2026-03-20T11:46:22.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.672+0000 7f4e96f52900 20 started zone id=4b634ebf-58f5-45bc-839a-cb4b425e773a (name=a1) with tier type = 2026-03-20T11:46:22.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.732+0000 7f4e96f52900 20 add_watcher() i=2 2026-03-20T11:46:22.734 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.738+0000 7f4e96f52900 20 add_watcher() i=4 2026-03-20T11:46:22.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.738+0000 7f4e96f52900 20 add_watcher() i=5 2026-03-20T11:46:22.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.738+0000 7f4e96f52900 20 add_watcher() i=6 2026-03-20T11:46:22.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.738+0000 7f4e96f52900 20 add_watcher() i=7 2026-03-20T11:46:22.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.738+0000 7f4e96f52900 20 add_watcher() i=0 2026-03-20T11:46:22.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.738+0000 7f4e96f52900 20 add_watcher() i=3 2026-03-20T11:46:22.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.738+0000 7f4e96f52900 20 add_watcher() i=1 2026-03-20T11:46:22.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.738+0000 7f4e96f52900 2 all 8 watchers are set, enabling cache 2026-03-20T11:46:22.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.741+0000 7f4e96f52900 20 rgw_check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2026-03-20T11:46:22.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.741+0000 7f4e96f52900 20 rgw_check_secure_mon_conn(): mode 1 is insecure 2026-03-20T11:46:22.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.741+0000 7f4e96f52900 5 note: GC not initialized 2026-03-20T11:46:22.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.741+0000 7f4e42fdd640 20 reqs_thread_entry: start 2026-03-20T11:46:22.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.798+0000 7f4e96f52900 20 init_complete bucket index max shards: 11 2026-03-20T11:46:22.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.798+0000 7f4e96f52900 20 Filter name: none 2026-03-20T11:46:22.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.798+0000 7f4e40fd9640 20 reqs_thread_entry: start 2026-03-20T11:46:22.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.798+0000 7f4df2ffd640 20 reqs_thread_entry: start 2026-03-20T11:46:22.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.800+0000 7f4e96f52900 10 rgw bucket trim manager: Watching bilog.trim 2026-03-20T11:46:22.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.800+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f904bc0:12BucketTrimCR: operate() 2026-03-20T11:46:22.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.800+0000 7f4e96f52900 10 fetching active bucket counters 2026-03-20T11:46:22.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.800+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f9072e0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:22.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.800+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f9072e0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:22.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.800+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is io blocked 2026-03-20T11:46:22.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.800+0000 7f4e6b7fe640 20 trim: get_bucket_counters: [] 2026-03-20T11:46:22.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.801+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f9072e0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:22.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.801+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f9072e0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:22.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.801+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f9072e0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:22.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.801+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f9072e0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:22.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.801+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f904bc0:12BucketTrimCR: operate() 2026-03-20T11:46:22.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.801+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f9086b0:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:22.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.801+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f9086b0:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:22.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.801+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is io blocked 2026-03-20T11:46:22.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.801+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f9086b0:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:22.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.801+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f9086b0:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:22.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.801+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f9086b0:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:22.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.801+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f9086b0:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:22.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.801+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f904bc0:12BucketTrimCR: operate() 2026-03-20T11:46:22.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.801+0000 7f4e96f52900 10 listing cold buckets from marker=12:9c77f2bb:root::dipepu-31:head 2026-03-20T11:46:22.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.801+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:14MetadataListCR: operate() 2026-03-20T11:46:22.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.801+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:14MetadataListCR: operate() 2026-03-20T11:46:22.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.801+0000 7f4e96f52900 20 enqueued request req=0x7f4e58002b50 2026-03-20T11:46:22.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.801+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:22.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.801+0000 7f4e96f52900 20 req: 0x7f4e58002b50 2026-03-20T11:46:22.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.801+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is io blocked 2026-03-20T11:46:22.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.801+0000 7f4e53fff640 20 dequeued request req=0x7f4e58002b50 2026-03-20T11:46:22.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.801+0000 7f4e53fff640 20 RGWWQ: empty 2026-03-20T11:46:22.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.801+0000 7f4e53fff640 20 rgw async rados processor: starting metadata listing at 12:9c77f2bb:root::dipepu-31:head 2026-03-20T11:46:22.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.805+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-31 2026-03-20T11:46:22.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.805+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:46:22.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.808+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-33 2026-03-20T11:46:22.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.808+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:22.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.812+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-4 2026-03-20T11:46:22.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.812+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:46:22.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.814+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:22.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.816+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:22.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.819+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-43 2026-03-20T11:46:22.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.819+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-30 2026-03-20T11:46:22.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.819+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-18 2026-03-20T11:46:22.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.819+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-2 2026-03-20T11:46:22.816 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.819+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:46:22.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.821+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-8 2026-03-20T11:46:22.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.821+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:46:22.820 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.823+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:22.821 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.825+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-58 2026-03-20T11:46:22.821 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.825+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:46:22.823 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.826+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:22.824 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.828+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-27 2026-03-20T11:46:22.824 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.828+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-28 2026-03-20T11:46:22.824 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.828+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-59 2026-03-20T11:46:22.824 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.828+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:46:22.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.829+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:22.827 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.830+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-40 2026-03-20T11:46:22.827 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.830+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-22 2026-03-20T11:46:22.827 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.830+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:46:22.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.832+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-56 2026-03-20T11:46:22.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.832+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:46:22.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.833+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:46:22.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.834+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:22.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.835+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-53 2026-03-20T11:46:22.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.835+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:46:22.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.836+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:46:22.834 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.837+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:46:22.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.838+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:46:22.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.839+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:46:22.837 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.840+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-34 2026-03-20T11:46:22.837 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.840+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-21 2026-03-20T11:46:22.837 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.840+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-14 2026-03-20T11:46:22.837 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.840+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-50 2026-03-20T11:46:22.837 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.840+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-57 2026-03-20T11:46:22.837 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.840+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:46:22.837 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.840+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-39 2026-03-20T11:46:22.837 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.840+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:22.837 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.840+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-37 2026-03-20T11:46:22.837 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.840+0000 7f4e53fff640 20 rgw async rados processor: restarting metadata listing 2026-03-20T11:46:22.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.848+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-10 2026-03-20T11:46:22.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.848+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:22.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.856+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:22.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.862+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:22.866 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.869+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:22.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.876+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:22.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.882+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:22.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.888+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-19 2026-03-20T11:46:22.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.888+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-3 2026-03-20T11:46:22.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.888+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:22.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.895+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:22.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.901+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-17 2026-03-20T11:46:22.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.901+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:22.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.908+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-38 2026-03-20T11:46:22.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.908+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:22.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.914+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-24 2026-03-20T11:46:22.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.914+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-36 2026-03-20T11:46:22.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.914+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:22.916 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.919+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:22.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.925+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:22.927 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.930+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-44 2026-03-20T11:46:22.927 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.930+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-35 2026-03-20T11:46:22.927 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.930+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:22.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.935+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-16 2026-03-20T11:46:22.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.935+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:22.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.940+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:22.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.944+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:22.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.950+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-55 2026-03-20T11:46:22.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.950+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-13 2026-03-20T11:46:22.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.950+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-48 2026-03-20T11:46:22.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.950+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-25 2026-03-20T11:46:22.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.950+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-51 2026-03-20T11:46:22.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.950+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:22.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.955+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:22.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.959+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-46 2026-03-20T11:46:22.956 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.959+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:46:22.960 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.964+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:46:22.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.968+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:46:22.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.972+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-26 2026-03-20T11:46:22.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.972+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-20 2026-03-20T11:46:22.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.972+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:46:22.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.976+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:46:22.977 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.980+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-1 2026-03-20T11:46:22.977 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.980+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:46:22.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.984+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:46:22.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e53fff640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:14MetadataListCR: operate() 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:14MetadataListCR: operate() 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:14MetadataListCR: operate() 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:14MetadataListCR: operate() 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f904bc0:12BucketTrimCR: operate() 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 4 collected 50 buckets for trim 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 4 starting trim on bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f909ae0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 4 starting trim on bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f90adb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 4 starting trim on bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f90c080:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 4 starting trim on bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f9086b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f90d350:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f90daa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f90e4c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f9086b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 enqueued request req=0x564c6f90ee70 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 req: 0x564c6f90ee70 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 run: stack=0x564c6f7e7650 is io blocked 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f90d350:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 enqueued request req=0x564c6f90f210 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 req: 0x564c6f90ee70 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 req: 0x564c6f90f210 2026-03-20T11:46:22.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is io blocked 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f90daa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 enqueued request req=0x564c6f90f630 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 req: 0x564c6f90ee70 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 req: 0x564c6f90f210 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 req: 0x564c6f90f630 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is io blocked 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f90e4c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 enqueued request req=0x564c6f81c970 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 req: 0x564c6f90ee70 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 req: 0x564c6f90f210 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 req: 0x564c6f90f630 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 req: 0x564c6f81c970 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e96f52900 20 run: stack=0x564c6f907840 is io blocked 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e537fe640 20 dequeued request req=0x564c6f90ee70 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e537fe640 20 RGWWQ: 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e537fe640 20 req: 0x564c6f90f210 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e537fe640 20 req: 0x564c6f90f630 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e537fe640 20 req: 0x564c6f81c970 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e537fe640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 : miss 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e537fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e517fa640 20 dequeued request req=0x564c6f90f210 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e517fa640 20 RGWWQ: 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e517fa640 20 req: 0x564c6f90f630 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e517fa640 20 req: 0x564c6f81c970 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e517fa640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 : miss 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e517fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e52ffd640 20 dequeued request req=0x564c6f90f630 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e52ffd640 20 RGWWQ: 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e52ffd640 20 req: 0x564c6f81c970 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e52ffd640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 : miss 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e52ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e51ffb640 20 dequeued request req=0x564c6f81c970 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e51ffb640 20 RGWWQ: empty 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e51ffb640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 : miss 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.988+0000 7f4e51ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e537fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e537fe640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 info.flags=0x17 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e537fe640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 to cache LRU end 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e537fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e537fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e537fe640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e537fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e537fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-47 : miss 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e537fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e51ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e51ffb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 info.flags=0x17 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e51ffb640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 to cache LRU end 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e51ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e51ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e51ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e51ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-20 : miss 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e51ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:22.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e517fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:22.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e517fa640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 info.flags=0x17 2026-03-20T11:46:22.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e52ffd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:22.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e517fa640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 to cache LRU end 2026-03-20T11:46:22.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e517fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:22.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e517fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:22.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e517fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:22.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e517fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-58 : miss 2026-03-20T11:46:22.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e517fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:22.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e52ffd640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 info.flags=0x17 2026-03-20T11:46:22.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e52ffd640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 to cache LRU end 2026-03-20T11:46:22.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e52ffd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:22.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e52ffd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:22.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e52ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:46:22.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e52ffd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-30 : miss 2026-03-20T11:46:22.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e52ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:22.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e537fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:22.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e537fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-47 info.flags=0x0 2026-03-20T11:46:22.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e537fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-47 to cache LRU end 2026-03-20T11:46:22.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e537fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-47 : miss 2026-03-20T11:46:22.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e537fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:22.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e51ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:22.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e51ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-20 info.flags=0x0 2026-03-20T11:46:22.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e51ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-20 to cache LRU end 2026-03-20T11:46:22.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e51ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-20 : miss 2026-03-20T11:46:22.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e51ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e51ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e51ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-20 info.flags=0x0 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e51ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-20 to cache LRU end 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e51ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25): adding source pipe: {s={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e51ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25): adding dest pipe: {s={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e51ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e51ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e51ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f90e4c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f90e4c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f90e4c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f90e4c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f90c080:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e517fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e517fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-58 info.flags=0x0 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e517fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-58 to cache LRU end 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f81cd90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e517fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-58 : miss 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e517fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f90c080:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 run: stack=0x564c6f907840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f90e4c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f90e4c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e52ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e52ffd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-30 info.flags=0x0 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e52ffd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-30 to cache LRU end 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e52ffd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-30 : miss 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e52ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e537fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e537fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-47 info.flags=0x0 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e537fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-47 to cache LRU end 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e537fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7): adding source pipe: {s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e537fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7): adding dest pipe: {s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e537fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e537fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:22.989 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:22 +0000 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e537fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:22 +0000 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114622Z 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:22 +0000 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114622Z 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 8e648b64800829c6f7d437ae92fa0d5599e26fd0083a6dc1fed640787ea5d2aa 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114622Z 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:8e648b64800829c6f7d437ae92fa0d5599e26fd0083a6dc1fed640787ea5d2aa 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 92d07bd875e1fbcb3bdfcda8575008a45b3e25bd6caa3dcc11ad12f68361b35a 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=92d07bd875e1fbcb3bdfcda8575008a45b3e25bd6caa3dcc11ad12f68361b35a 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114622Z 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=0, curl_handle=0x564c6f81eb70 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is io blocked 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f9086b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f9086b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f9086b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f9086b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f8210e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4df2ffd640 20 link_request req_data=0x564c6f81f660 req_data->id=0, curl_handle=0x564c6f81eb70 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 run: stack=0x564c6f7e7650 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f821890:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f821890:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:22 +0000 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:22 +0000 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114622Z 2026-03-20T11:46:22.990 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.989+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:22 +0000 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114622Z 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 52e7ecc45024bfd091c4d0ddede970100faeb0abf532864a4ee91e607a91f188 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114622Z 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:52e7ecc45024bfd091c4d0ddede970100faeb0abf532864a4ee91e607a91f188 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 749ed1ed4b2148c452e3361551660782328d9cfdf97f67f82905925597e3b765 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=749ed1ed4b2148c452e3361551660782328d9cfdf97f67f82905925597e3b765 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114622Z 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=1, curl_handle=0x564c6f823420 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is io blocked 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4df2ffd640 20 link_request req_data=0x564c6f823950 req_data->id=1, curl_handle=0x564c6f823420 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e517fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e517fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-58 info.flags=0x0 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e517fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-58 to cache LRU end 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e517fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18): adding source pipe: {s={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e517fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18): adding dest pipe: {s={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e517fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e517fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e517fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e52ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e52ffd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-30 info.flags=0x0 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e52ffd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-30 to cache LRU end 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f90d350:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f90d350:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e52ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35): adding source pipe: {s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f90d350:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f90d350:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f909ae0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e52ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35): adding dest pipe: {s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f90fa50:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f909ae0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f910070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f910070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e52ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e52ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:22 +0000 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e52ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:22 +0000 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:22.991 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114622Z 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:22 +0000 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114622Z 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = eff6976a6771f269355ef4300212cb3ae1726fd18c35d6622bb2289f8fe79317 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114622Z 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:eff6976a6771f269355ef4300212cb3ae1726fd18c35d6622bb2289f8fe79317 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 97653867f671aeb439a6f557db53dfff95c61595c5e4238cb75c4334226f5573 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=97653867f671aeb439a6f557db53dfff95c61595c5e4238cb75c4334226f5573 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114622Z 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=2, curl_handle=0x564c6f911bf0 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 run: stack=0x564c6f90d800 is io blocked 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f90daa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f90daa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f90daa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f90daa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f90adb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4df2ffd640 20 link_request req_data=0x564c6f912300 req_data->id=2, curl_handle=0x564c6f911bf0 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 cr:s=0x564c6f913fc0:op=0x564c6f9139a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f90adb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 cr:s=0x564c6f913fc0:op=0x564c6f914150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 cr:s=0x564c6f913fc0:op=0x564c6f914150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:22 +0000 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:22 +0000 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114622Z 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:22 +0000 2026-03-20T11:46:22.992 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:22.993 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.993 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114622Z 2026-03-20T11:46:22.993 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:22.993 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:22.993 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 4420f64432564e2b70721c19c432b814bf1ea374ad6ee33c9e86a10980b10227 2026-03-20T11:46:22.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:22.993 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114622Z 2026-03-20T11:46:22.993 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:22.993 INFO:teuthology.orchestra.run.vm02.stderr:4420f64432564e2b70721c19c432b814bf1ea374ad6ee33c9e86a10980b10227 2026-03-20T11:46:22.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:22.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:22.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:22.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:22.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = f07f28e00cf7291f42c4f1d99c8ce5ea15531cfcba216035f001e53bf4995624 2026-03-20T11:46:22.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=f07f28e00cf7291f42c4f1d99c8ce5ea15531cfcba216035f001e53bf4995624 2026-03-20T11:46:22.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114622Z 2026-03-20T11:46:22.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:22.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=3, curl_handle=0x564c6f915d30 2026-03-20T11:46:22.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4e96f52900 20 run: stack=0x564c6f913fc0 is io blocked 2026-03-20T11:46:22.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.990+0000 7f4df2ffd640 20 link_request req_data=0x564c6f916440 req_data->id=3, curl_handle=0x564c6f915d30 2026-03-20T11:46:22.993 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:22.990+0000 7fd0c0bcf640 -1 req 15159588254386262535 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:22.993 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:22.990+0000 7fd0b63ba640 -1 req 18342715408666824652 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:22.993 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:22.994+0000 7fd07733c640 -1 req 8381712835861306327 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f821890:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f821890:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f8210e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 15 stack 0x564c6f821700 end 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is done 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 collect(): s=0x564c6f7e7650 stack=0x564c6f821700 is complete 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 15 stack 0x564c6f7e7650 end 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 stack->operate() returned ret=-22 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 run: stack=0x564c6f7e7650 is done 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 cr:s=0x564c6f913fc0:op=0x564c6f914150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 cr:s=0x564c6f913fc0:op=0x564c6f914150:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 cr:s=0x564c6f913fc0:op=0x564c6f9139a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 15 stack 0x564c6f913fc0 end 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 run: stack=0x564c6f913fc0 is done 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 4 starting trim on bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f90adb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 collect(): s=0x564c6f8132d0 stack=0x564c6f913fc0 is complete 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f90adb0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 15 stack 0x564c6f8132d0 end 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 stack->operate() returned ret=-22 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is done 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f914150:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f914150:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 enqueued request req=0x564c6f7e7460 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 req: 0x564c6f7e7460 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 run: stack=0x564c6f7e7650 is io blocked 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f90adb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 4 starting trim on bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f821840:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f821840:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 enqueued request req=0x564c6f90ee70 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 req: 0x564c6f7e7460 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 req: 0x564c6f90ee70 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.991+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is io blocked 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.992+0000 7f4e50ff9640 20 dequeued request req=0x564c6f7e7460 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.992+0000 7f4e50ff9640 20 RGWWQ: 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.992+0000 7f4e50ff9640 20 req: 0x564c6f90ee70 2026-03-20T11:46:22.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.992+0000 7f4e50ff9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 : miss 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.992+0000 7f4e50ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.992+0000 7f4e507f8640 20 dequeued request req=0x564c6f90ee70 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.992+0000 7f4e507f8640 20 RGWWQ: empty 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.992+0000 7f4e507f8640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 : miss 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.992+0000 7f4e507f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f90e4c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e50ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e50ff9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 info.flags=0x17 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e50ff9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 to cache LRU end 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e50ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e50ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e50ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e50ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-52 : miss 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e50ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f90e4c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f910070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f90e4c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f910070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f90e4c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f910070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f81cd90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 15 stack 0x564c6f907e20 end 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is done 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f910070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f90c080:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 collect(): s=0x564c6f907840 stack=0x564c6f907e20 is complete 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 10 trimming bilogs for bucket=:dipepu-20[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25]) markers=[,,,,,,,,00000000005.42.14,,], shards=11 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f90fa50:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 15 stack 0x564c6f90d800 end 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 run: stack=0x564c6f90d800 is done 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f81e210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 10 trimming bilog shard 8 of :dipepu-20[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25]) at marker 00000000005.42.14 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 run: stack=0x564c6f907840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f909ae0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 collect(): s=0x564c6f93f450 stack=0x564c6f90d800 is complete 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 10 trimming bilogs for bucket=:dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) markers=[,00000000001.77.6,,,,00000000001.65.6,,,00000000001.20.6,,00000000001.28.6], shards=11 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f917ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f90e4c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 10 trimming bilog shard 1 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.77.6 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 10 trimming bilog shard 5 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.65.6 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 10 trimming bilog shard 8 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.20.6 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 10 trimming bilog shard 10 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.28.6 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f917ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25.8 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is io blocked 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f94f810:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f947990:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 cr:s=0x564c6f9514c0:op=0x564c6f9540f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 cr:s=0x564c6f947140:op=0x564c6f91a010:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f94f810:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.1 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 run: stack=0x564c6f90d800 is io blocked 2026-03-20T11:46:22.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f947990:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.5 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is io blocked 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 cr:s=0x564c6f9514c0:op=0x564c6f9540f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.8 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 run: stack=0x564c6f9514c0 is io blocked 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 cr:s=0x564c6f947140:op=0x564c6f91a010:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.10 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e96f52900 20 run: stack=0x564c6f947140 is io blocked 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e507f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e507f8640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 info.flags=0x17 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e507f8640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 to cache LRU end 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e507f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e507f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.993+0000 7f4e507f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.994+0000 7f4e507f8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-42 : miss 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.994+0000 7f4e507f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.994+0000 7f4e50ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.994+0000 7f4e50ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-52 info.flags=0x0 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.994+0000 7f4e50ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-52 to cache LRU end 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.994+0000 7f4e507f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.994+0000 7f4e507f8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-42 info.flags=0x0 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.994+0000 7f4e507f8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-42 to cache LRU end 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.994+0000 7f4e50ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-52 : miss 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.994+0000 7f4e507f8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-42 : miss 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.994+0000 7f4e50ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.994+0000 7f4e507f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e50ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e50ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-52 info.flags=0x0 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e50ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-52 to cache LRU end 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e507f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e507f8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-42 info.flags=0x0 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e507f8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-42 to cache LRU end 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e507f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2): adding source pipe: {s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e50ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12): adding source pipe: {s={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e507f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2): adding dest pipe: {s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e50ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12): adding dest pipe: {s={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e507f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e50ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e507f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e50ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e507f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e50ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f821840:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f821840:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f914150:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f821840:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f914150:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f821840:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f914150:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f90adb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f914150:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f94ac40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f90adb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f94efc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f914150:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 run: stack=0x564c6f7e7650 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f94efc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:22 +0000 2026-03-20T11:46:22.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:22 +0000 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114622Z 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:22 +0000 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114622Z 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = ed515140ee2d0e2a286c1ea416c288e99ae9325077527147b74b795f75cb60c7 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114622Z 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:ed515140ee2d0e2a286c1ea416c288e99ae9325077527147b74b795f75cb60c7 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 0b70c296bc98b78437683cd7413f648989eefe512572c0f3a1d200bdde2fb6bf 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=0b70c296bc98b78437683cd7413f648989eefe512572c0f3a1d200bdde2fb6bf 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114622Z 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=4, curl_handle=0x564c6f911bf0 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 run: stack=0x564c6f8210c0 is io blocked 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f956560:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f956560:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4df2ffd640 20 link_request req_data=0x564c6f912300 req_data->id=4, curl_handle=0x564c6f911bf0 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:22 +0000 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:22 +0000 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114622Z 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:22 +0000 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114622Z 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = bbd36d38b4b5591def18df4b1ac72c93920ded2dcd82efe1fa25958abd3e608a 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:22.997 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114622Z 2026-03-20T11:46:22.998 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:22.998 INFO:teuthology.orchestra.run.vm02.stderr:bbd36d38b4b5591def18df4b1ac72c93920ded2dcd82efe1fa25958abd3e608a 2026-03-20T11:46:22.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:22.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:22.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:22.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:22.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 549520726332e9c9f8c259e4bdf79df24bf6a35a7b5056e56e1ac2ce570cd975 2026-03-20T11:46:22.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=549520726332e9c9f8c259e4bdf79df24bf6a35a7b5056e56e1ac2ce570cd975 2026-03-20T11:46:22.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:22.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114622Z 2026-03-20T11:46:22.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:22.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=5, curl_handle=0x564c6f81eb70 2026-03-20T11:46:22.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4df2ffd640 20 link_request req_data=0x564c6f81f660 req_data->id=5, curl_handle=0x564c6f81eb70 2026-03-20T11:46:22.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.995+0000 7f4e96f52900 20 run: stack=0x564c6f94b500 is io blocked 2026-03-20T11:46:22.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.998+0000 7f4e96f52900 20 cr:s=0x564c6f9514c0:op=0x564c6f9540f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:22.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.998+0000 7f4e96f52900 20 cr:s=0x564c6f9514c0:op=0x564c6f9540f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:22.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.998+0000 7f4e96f52900 15 stack 0x564c6f9514c0 end 2026-03-20T11:46:22.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.998+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:22.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.998+0000 7f4e96f52900 20 run: stack=0x564c6f9514c0 is done 2026-03-20T11:46:22.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.998+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f90e4c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:22.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:22.998+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:22.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 cr:s=0x564c6f947140:op=0x564c6f91a010:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 cr:s=0x564c6f947140:op=0x564c6f91a010:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 15 stack 0x564c6f947140 end 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 run: stack=0x564c6f947140 is done 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f90e4c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f94f810:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f94f810:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 15 stack 0x564c6f90d800 end 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 run: stack=0x564c6f90d800 is done 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f947990:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f947990:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 15 stack 0x564c6f821700 end 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is done 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f917ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f917ae0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 15 stack 0x564c6f907e20 end 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is done 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f90e4c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f81e210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f90e4c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f90c080:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 trim: trimmed bucket instance dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 15 stack 0x564c6f907840 end 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 run: stack=0x564c6f907840 is done 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f909ae0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 trim: trimmed bucket instance dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 15 stack 0x564c6f93f450 end 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is done 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f909ae0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 4 starting trim on bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f917ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 4 starting trim on bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f917ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 enqueued request req=0x564c6f81f1b0 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 req: 0x564c6f81f1b0 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is io blocked 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f94f7e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f94f7e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 enqueued request req=0x564c6f81c970 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 req: 0x564c6f81f1b0 2026-03-20T11:46:22.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 req: 0x564c6f81c970 2026-03-20T11:46:23.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e96f52900 20 run: stack=0x564c6f907840 is io blocked 2026-03-20T11:46:23.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e4fff7640 20 dequeued request req=0x564c6f81f1b0 2026-03-20T11:46:23.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e4fff7640 20 RGWWQ: 2026-03-20T11:46:23.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e4fff7640 20 req: 0x564c6f81c970 2026-03-20T11:46:23.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e4fff7640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 : miss 2026-03-20T11:46:23.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e4fff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e4f7f6640 20 dequeued request req=0x564c6f81c970 2026-03-20T11:46:23.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e4f7f6640 20 RGWWQ: empty 2026-03-20T11:46:23.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e4f7f6640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 : miss 2026-03-20T11:46:23.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e4f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e4fff7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:46:23.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e4fff7640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 info.flags=0x17 2026-03-20T11:46:23.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e4fff7640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 to cache LRU end 2026-03-20T11:46:23.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e4fff7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e4fff7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e4fff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:46:23.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e4fff7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-2 : miss 2026-03-20T11:46:23.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e4fff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e4f7f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e4f7f6640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 info.flags=0x17 2026-03-20T11:46:23.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e4f7f6640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 to cache LRU end 2026-03-20T11:46:23.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e4f7f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e4f7f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e4f7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:23.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e4f7f6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-57 : miss 2026-03-20T11:46:23.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.001+0000 7f4e4f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.000 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:23.002+0000 7fd091370640 -1 req 3359142042425461509 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e4fff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e4fff7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-2 info.flags=0x0 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e4fff7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-2 to cache LRU end 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e4fff7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-2 : miss 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e4fff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e4f7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e4f7f6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-57 info.flags=0x0 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e4f7f6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-57 to cache LRU end 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e4f7f6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-57 : miss 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e4f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e4f7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e4f7f6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-57 info.flags=0x0 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e4f7f6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-57 to cache LRU end 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e4f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17): adding source pipe: {s={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e4f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17): adding dest pipe: {s={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e4f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e4f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e4f7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e4fff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e4fff7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-2 info.flags=0x0 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e4fff7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-2 to cache LRU end 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e4fff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2): adding source pipe: {s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e4fff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2): adding dest pipe: {s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e4fff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e4fff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e4fff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f94f7e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f94f7e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f917ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f94f7e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f917ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f94f7e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f917ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f917ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f9113b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 run: stack=0x564c6f907840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f909ae0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f822070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f90e450:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f909ae0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.001 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f822070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = aa7b5d87cd902aa02cf859fdd331d00efeb4c909fa57be4a69b18d0a88e40309 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:aa7b5d87cd902aa02cf859fdd331d00efeb4c909fa57be4a69b18d0a88e40309 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = b3935a4b325693727d32ca04fdb4485305d29a1f3f2f09ae2a611e02ff3e9a9c 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b3935a4b325693727d32ca04fdb4485305d29a1f3f2f09ae2a611e02ff3e9a9c 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=6, curl_handle=0x564c6f915d30 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is io blocked 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f917ae0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f917ae0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4df2ffd640 20 link_request req_data=0x564c6f916440 req_data->id=6, curl_handle=0x564c6f915d30 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.002 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.003 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.003 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.003 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.003 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.003 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.003 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.003 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = fbb20be9446d15487ce3db63d4920bc487bcd642649285a40416cb2de48e37b4 2026-03-20T11:46:23.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.003 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.003 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.003 INFO:teuthology.orchestra.run.vm02.stderr:fbb20be9446d15487ce3db63d4920bc487bcd642649285a40416cb2de48e37b4 2026-03-20T11:46:23.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 91b66929de631334223221e94d2e1924b800976331fd084311b84ea4e1505499 2026-03-20T11:46:23.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=91b66929de631334223221e94d2e1924b800976331fd084311b84ea4e1505499 2026-03-20T11:46:23.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=7, curl_handle=0x564c6f823420 2026-03-20T11:46:23.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is io blocked 2026-03-20T11:46:23.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.002+0000 7f4df2ffd640 20 link_request req_data=0x564c6f823950 req_data->id=7, curl_handle=0x564c6f823420 2026-03-20T11:46:23.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f94efc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f94efc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:23.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f94ac40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e96f52900 15 stack 0x564c6f8210c0 end 2026-03-20T11:46:23.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e96f52900 20 run: stack=0x564c6f8210c0 is done 2026-03-20T11:46:23.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f90adb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e96f52900 20 collect(): s=0x564c6f8132d0 stack=0x564c6f8210c0 is complete 2026-03-20T11:46:23.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e96f52900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:23.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e96f52900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:23.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f90adb0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:23.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e96f52900 15 stack 0x564c6f8132d0 end 2026-03-20T11:46:23.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e96f52900 20 stack->operate() returned ret=-22 2026-03-20T11:46:23.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is done 2026-03-20T11:46:23.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e96f52900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:23.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e96f52900 4 starting trim on bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:46:23.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f9540f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f9540f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e96f52900 20 enqueued request req=0x564c6f90e1f0 2026-03-20T11:46:23.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e96f52900 20 req: 0x564c6f90e1f0 2026-03-20T11:46:23.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is io blocked 2026-03-20T11:46:23.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e4eff5640 20 dequeued request req=0x564c6f90e1f0 2026-03-20T11:46:23.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e4eff5640 20 RGWWQ: empty 2026-03-20T11:46:23.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e4eff5640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 : miss 2026-03-20T11:46:23.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e4eff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e4eff5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e4eff5640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 info.flags=0x17 2026-03-20T11:46:23.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e4eff5640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 to cache LRU end 2026-03-20T11:46:23.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e4eff5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e4eff5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:23.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e4eff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:46:23.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e4eff5640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-29 : miss 2026-03-20T11:46:23.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.036+0000 7f4e4eff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f956560:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f956560:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f956560:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f956560:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f914150:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 15 stack 0x564c6f94b500 end 2026-03-20T11:46:23.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 run: stack=0x564c6f94b500 is done 2026-03-20T11:46:23.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.034 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 collect(): s=0x564c6f7e7650 stack=0x564c6f94b500 is complete 2026-03-20T11:46:23.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 10 trimming bilogs for bucket=:dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) markers=[00000000004.19.6,00000000002.34.1,00000000004.47.6,00000000002.22.1,00000000004.31.6,00000000002.67.1,00000000002.32.1,00000000004.16.6,00000000002.26.1,00000000002.76.1,00000000002.54.1], shards=11 2026-03-20T11:46:23.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f81e210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 10 trimming bilog shard 0 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000004.19.6 2026-03-20T11:46:23.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 10 trimming bilog shard 1 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.34.1 2026-03-20T11:46:23.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 10 trimming bilog shard 2 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000004.47.6 2026-03-20T11:46:23.035 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 10 trimming bilog shard 3 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.22.1 2026-03-20T11:46:23.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 10 trimming bilog shard 4 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000004.31.6 2026-03-20T11:46:23.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 10 trimming bilog shard 5 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.67.1 2026-03-20T11:46:23.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 10 trimming bilog shard 6 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.32.1 2026-03-20T11:46:23.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 10 trimming bilog shard 7 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000004.16.6 2026-03-20T11:46:23.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 10 trimming bilog shard 8 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.26.1 2026-03-20T11:46:23.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 10 trimming bilog shard 9 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.76.1 2026-03-20T11:46:23.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 10 trimming bilog shard 10 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.54.1 2026-03-20T11:46:23.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 run: stack=0x564c6f7e7650 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f821840:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f919dc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f49f9b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f948720:op=0x564c6f90adb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f9139a0:op=0x564c6f4a0310:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f918ea0:op=0x564c6f4a4140:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f9155e0:op=0x564c6f81ccf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f951b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f952370:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f919040:op=0x564c6f4a0df0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f94a930:op=0x564c6f4a15f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f821840:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.0 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 run: stack=0x564c6f94b500 is io blocked 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f919dc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.1 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 run: stack=0x564c6f8210c0 is io blocked 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f49f9b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.2 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 run: stack=0x564c6f90d800 is io blocked 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f948720:op=0x564c6f90adb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.3 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 run: stack=0x564c6f948720 is io blocked 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f9139a0:op=0x564c6f4a0310:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.4 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 run: stack=0x564c6f9139a0 is io blocked 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f918ea0:op=0x564c6f4a4140:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.5 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 run: stack=0x564c6f918ea0 is io blocked 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f9155e0:op=0x564c6f81ccf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.6 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 run: stack=0x564c6f9155e0 is io blocked 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f951b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.7 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is io blocked 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f952370:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.8 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 run: stack=0x564c6f953ef0 is io blocked 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f919040:op=0x564c6f4a0df0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.9 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 run: stack=0x564c6f919040 is io blocked 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 cr:s=0x564c6f94a930:op=0x564c6f4a15f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.10 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.037+0000 7f4e96f52900 20 run: stack=0x564c6f94a930 is io blocked 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.038+0000 7f4e4eff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.038+0000 7f4e4eff5640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-29 info.flags=0x0 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.038+0000 7f4e4eff5640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-29 to cache LRU end 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.038+0000 7f4e4eff5640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-29 : miss 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.038+0000 7f4e4eff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 cr:s=0x564c6f9155e0:op=0x564c6f81ccf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 cr:s=0x564c6f9155e0:op=0x564c6f81ccf0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 15 stack 0x564c6f9155e0 end 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 run: stack=0x564c6f9155e0 is done 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f81e210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 run: stack=0x564c6f7e7650 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 cr:s=0x564c6f948720:op=0x564c6f90adb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 cr:s=0x564c6f948720:op=0x564c6f90adb0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 15 stack 0x564c6f948720 end 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 run: stack=0x564c6f948720 is done 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f81e210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 run: stack=0x564c6f7e7650 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f952370:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f952370:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 15 stack 0x564c6f953ef0 end 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 run: stack=0x564c6f953ef0 is done 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f81e210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 run: stack=0x564c6f7e7650 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f821840:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f821840:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 15 stack 0x564c6f94b500 end 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 run: stack=0x564c6f94b500 is done 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f81e210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 run: stack=0x564c6f7e7650 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f951b70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f951b70:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 15 stack 0x564c6f94bd80 end 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is done 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f81e210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 run: stack=0x564c6f7e7650 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e4eff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e4eff5640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-29 info.flags=0x0 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e4eff5640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-29 to cache LRU end 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f49f9b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e4eff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34): adding source pipe: {s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f49f9b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 15 stack 0x564c6f90d800 end 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 run: stack=0x564c6f90d800 is done 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f81e210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e96f52900 20 run: stack=0x564c6f7e7650 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.039+0000 7f4e4eff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34): adding dest pipe: {s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f919dc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e4eff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e4eff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f919dc0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 15 stack 0x564c6f8210c0 end 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 run: stack=0x564c6f8210c0 is done 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f81e210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 run: stack=0x564c6f7e7650 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f918ea0:op=0x564c6f4a4140:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f918ea0:op=0x564c6f4a4140:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 15 stack 0x564c6f918ea0 end 2026-03-20T11:46:23.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e4eff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 run: stack=0x564c6f918ea0 is done 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f81e210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 run: stack=0x564c6f7e7650 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f9540f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f94a930:op=0x564c6f4a15f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f94a930:op=0x564c6f4a15f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 15 stack 0x564c6f94a930 end 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 run: stack=0x564c6f94a930 is done 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f9540f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f81e210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 run: stack=0x564c6f7e7650 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f9540f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f9540f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f94ac40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f821840:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f821840:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 1a153cd637e11f0cf6b991b6f5bca0c153c3180cf5ceb723df6a8775851f932d 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:1a153cd637e11f0cf6b991b6f5bca0c153c3180cf5ceb723df6a8775851f932d 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 465a43f1d3e59b966e3515482ea3e8e82678e84881447f15cff093a03099c764 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=465a43f1d3e59b966e3515482ea3e8e82678e84881447f15cff093a03099c764 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=8, curl_handle=0x564c6f81eb70 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 run: stack=0x564c6f8210c0 is io blocked 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4df2ffd640 20 link_request req_data=0x564c6f81f660 req_data->id=8, curl_handle=0x564c6f81eb70 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f919040:op=0x564c6f4a0df0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f919040:op=0x564c6f4a0df0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 15 stack 0x564c6f919040 end 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 run: stack=0x564c6f919040 is done 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f81e210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 run: stack=0x564c6f7e7650 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f9139a0:op=0x564c6f4a0310:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f9139a0:op=0x564c6f4a0310:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 15 stack 0x564c6f9139a0 end 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 run: stack=0x564c6f9139a0 is done 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f81e210:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f7e7650:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 trim: trimmed bucket instance dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 15 stack 0x564c6f7e7650 end 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 run: stack=0x564c6f7e7650 is done 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f919040:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 4 starting trim on bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f919040:op=0x564c6f90adb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 cr:s=0x564c6f919040:op=0x564c6f90adb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 enqueued request req=0x564c6f94ce80 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 req: 0x564c6f94ce80 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e96f52900 20 run: stack=0x564c6f919040 is io blocked 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e4e7f4640 20 dequeued request req=0x564c6f94ce80 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e4e7f4640 20 RGWWQ: empty 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e4e7f4640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 : miss 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e4e7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e4e7f4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e4e7f4640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 info.flags=0x17 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e4e7f4640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 to cache LRU end 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e4e7f4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e4e7f4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e4e7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e4e7f4640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-23 : miss 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.040+0000 7f4e4e7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e4e7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e4e7f4640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-23 info.flags=0x0 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e4e7f4640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-23 to cache LRU end 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e4e7f4640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-23 : miss 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e4e7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e4e7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e4e7f4640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-23 info.flags=0x0 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e4e7f4640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-23 to cache LRU end 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e4e7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28): adding source pipe: {s={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e4e7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28): adding dest pipe: {s={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e4e7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e4e7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e4e7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 20 cr:s=0x564c6f919040:op=0x564c6f90adb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 20 cr:s=0x564c6f919040:op=0x564c6f90adb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 20 cr:s=0x564c6f919040:op=0x564c6f90adb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 20 cr:s=0x564c6f919040:op=0x564c6f90adb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 20 cr:s=0x564c6f919040:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f914120:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 20 cr:s=0x564c6f919040:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 20 run: stack=0x564c6f919040 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f90adb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f90adb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 9348ee969b3f1be442f7326cd3f11a38e644641bfd9799ee6f026b32df90afc8 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:9348ee969b3f1be442f7326cd3f11a38e644641bfd9799ee6f026b32df90afc8 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 8381f09d569e54153d04d8c73e4a1523ab97618eef52c84d4d0f41930b728440 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=8381f09d569e54153d04d8c73e4a1523ab97618eef52c84d4d0f41930b728440 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=9, curl_handle=0x564c6f911bf0 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4e96f52900 20 run: stack=0x564c6f90d800 is io blocked 2026-03-20T11:46:23.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.041+0000 7f4df2ffd640 20 link_request req_data=0x564c6f912300 req_data->id=9, curl_handle=0x564c6f911bf0 2026-03-20T11:46:23.043 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:23.039+0000 7fcfe2212640 -1 req 8342707654133553242 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:23.043 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:23.044+0000 7fcfc29d3640 -1 req 2902651141746476244 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:23.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f917ae0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f917ae0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:23.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f90e450:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e96f52900 15 stack 0x564c6f907e20 end 2026-03-20T11:46:23.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is done 2026-03-20T11:46:23.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f909ae0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e96f52900 20 collect(): s=0x564c6f93f450 stack=0x564c6f907e20 is complete 2026-03-20T11:46:23.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e96f52900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:23.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e96f52900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:23.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f909ae0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:23.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e96f52900 15 stack 0x564c6f93f450 end 2026-03-20T11:46:23.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e96f52900 20 stack->operate() returned ret=-22 2026-03-20T11:46:23.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is done 2026-03-20T11:46:23.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e96f52900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:23.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f909ae0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e96f52900 4 starting trim on bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:46:23.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f947990:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f947990:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e96f52900 20 enqueued request req=0x564c6f81c970 2026-03-20T11:46:23.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e96f52900 20 req: 0x564c6f81c970 2026-03-20T11:46:23.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is io blocked 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e4dff3640 20 dequeued request req=0x564c6f81c970 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e4dff3640 20 RGWWQ: empty 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e4dff3640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 : miss 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e4dff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e4dff3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e4dff3640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 info.flags=0x17 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e4dff3640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 to cache LRU end 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e4dff3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e4dff3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e4dff3640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e4dff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e4dff3640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-34 : miss 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e4dff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e4dff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e4dff3640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-34 info.flags=0x0 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e4dff3640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-34 to cache LRU end 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e4dff3640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-34 : miss 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e4dff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e4dff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e4dff3640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-34 info.flags=0x0 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e4dff3640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-34 to cache LRU end 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e4dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39): adding source pipe: {s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.044+0000 7f4e4dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39): adding dest pipe: {s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e4dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e4dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e4dff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f947990:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f947990:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f947990:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f947990:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f909ae0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f917ae0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f909ae0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f90e450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f90e450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 0ade9f2300e693722c970349244ad517b7471fd9cca8a334947adb98391222d2 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:0ade9f2300e693722c970349244ad517b7471fd9cca8a334947adb98391222d2 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = be4b4e1180f1c570944de4810980f6e5700fb99f6466d109be6bcb781d5c54dd 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=be4b4e1180f1c570944de4810980f6e5700fb99f6466d109be6bcb781d5c54dd 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=10, curl_handle=0x564c6f823420 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is io blocked 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f822070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f822070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f822070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f822070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f9113b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 15 stack 0x564c6f821700 end 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is done 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 collect(): s=0x564c6f907840 stack=0x564c6f821700 is complete 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 10 trimming bilogs for bucket=:dipepu-57[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17]) markers=[,00000000001.41.6,,,,00000000001.23.6,,,,,], shards=11 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f9113b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 10 trimming bilog shard 1 of :dipepu-57[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17]) at marker 00000000001.41.6 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 10 trimming bilog shard 5 of :dipepu-57[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17]) at marker 00000000001.23.6 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 run: stack=0x564c6f907840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f94f7e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f49f9b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f94f7e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17.1 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is io blocked 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f49f9b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17.5 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is io blocked 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.045+0000 7f4df2ffd640 20 link_request req_data=0x564c6f918e60 req_data->id=10, curl_handle=0x564c6f823420 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f49f9b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f49f9b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e96f52900 15 stack 0x564c6f94bd80 end 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is done 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f9113b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e96f52900 20 run: stack=0x564c6f907840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f94f7e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f94f7e0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e96f52900 15 stack 0x564c6f821700 end 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is done 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f9113b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e96f52900 20 trim: trimmed bucket instance dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:23.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e96f52900 15 stack 0x564c6f907840 end 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e96f52900 20 run: stack=0x564c6f907840 is done 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e96f52900 4 starting trim on bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f821e70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f821e70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e96f52900 20 enqueued request req=0x564c6f90ec10 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e96f52900 20 req: 0x564c6f90ec10 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e96f52900 20 run: stack=0x564c6f907840 is io blocked 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e4d7f2640 20 dequeued request req=0x564c6f90ec10 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e4d7f2640 20 RGWWQ: empty 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e4d7f2640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 : miss 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e4d7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e4d7f2640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e4d7f2640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 info.flags=0x17 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e4d7f2640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 to cache LRU end 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e4d7f2640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e4d7f2640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e4d7f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e4d7f2640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-54 : miss 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.046+0000 7f4e4d7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e4d7f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e4d7f2640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-54 info.flags=0x0 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e4d7f2640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-54 to cache LRU end 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e4d7f2640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-54 : miss 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e4d7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e4d7f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e4d7f2640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-54 info.flags=0x0 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e4d7f2640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-54 to cache LRU end 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e4d7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14): adding source pipe: {s={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e4d7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14): adding dest pipe: {s={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e4d7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e4d7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e4d7f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f821e70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f821e70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f821e70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f821e70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f9113b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 20 run: stack=0x564c6f907840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f9524f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f9524f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.047 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.048 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.048 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.048 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.048 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = a79ec22202d81c4993c7343b00985a85833641541de09ecd5adcbf77625afeeb 2026-03-20T11:46:23.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.048 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.048 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.048 INFO:teuthology.orchestra.run.vm02.stderr:a79ec22202d81c4993c7343b00985a85833641541de09ecd5adcbf77625afeeb 2026-03-20T11:46:23.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = b7b24e2812bb5dd4256f1febe3cb268d7f87e89b3259a8d52b024c4c9315bc77 2026-03-20T11:46:23.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b7b24e2812bb5dd4256f1febe3cb268d7f87e89b3259a8d52b024c4c9315bc77 2026-03-20T11:46:23.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=11, curl_handle=0x564c6f915d30 2026-03-20T11:46:23.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is io blocked 2026-03-20T11:46:23.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.047+0000 7f4df2ffd640 20 link_request req_data=0x564c6f908e00 req_data->id=11, curl_handle=0x564c6f915d30 2026-03-20T11:46:23.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f821840:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f821840:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:23.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f94ac40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e96f52900 15 stack 0x564c6f8210c0 end 2026-03-20T11:46:23.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e96f52900 20 run: stack=0x564c6f8210c0 is done 2026-03-20T11:46:23.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e96f52900 20 collect(): s=0x564c6f8132d0 stack=0x564c6f8210c0 is complete 2026-03-20T11:46:23.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e96f52900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:23.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e96f52900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:23.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:23.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e96f52900 15 stack 0x564c6f8132d0 end 2026-03-20T11:46:23.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e96f52900 20 stack->operate() returned ret=-22 2026-03-20T11:46:23.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is done 2026-03-20T11:46:23.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e96f52900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:23.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e96f52900 4 starting trim on bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:46:23.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f94ac40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f94ac40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e96f52900 20 enqueued request req=0x564c6f94c560 2026-03-20T11:46:23.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e96f52900 20 req: 0x564c6f94c560 2026-03-20T11:46:23.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is io blocked 2026-03-20T11:46:23.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e4cff1640 20 dequeued request req=0x564c6f94c560 2026-03-20T11:46:23.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e4cff1640 20 RGWWQ: empty 2026-03-20T11:46:23.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e4cff1640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 : miss 2026-03-20T11:46:23.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e4cff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e4cff1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e4cff1640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 info.flags=0x17 2026-03-20T11:46:23.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e4cff1640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 to cache LRU end 2026-03-20T11:46:23.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e4cff1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e4cff1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e4cff1640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:46:23.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e4cff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:46:23.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e4cff1640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-32 : miss 2026-03-20T11:46:23.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.081+0000 7f4e4cff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e4cff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e4cff1640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-32 info.flags=0x0 2026-03-20T11:46:23.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e4cff1640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-32 to cache LRU end 2026-03-20T11:46:23.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e4cff1640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-32 : miss 2026-03-20T11:46:23.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e4cff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e4cff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e4cff1640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-32 info.flags=0x0 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e4cff1640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-32 to cache LRU end 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e4cff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37): adding source pipe: {s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e4cff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37): adding dest pipe: {s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e4cff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e4cff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e4cff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f94ac40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f94ac40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f94ac40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f94ac40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f821840:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f910820:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f910820:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 385f5ac881f65fc344b610c1dcb9cbfe80b385db0e6c9412cb3094f259c3fa05 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:385f5ac881f65fc344b610c1dcb9cbfe80b385db0e6c9412cb3094f259c3fa05 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 83687fae3eed5d9ede3b691a32e2e302781fc5a573acd2c14fcfb2aa85c6ce29 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=83687fae3eed5d9ede3b691a32e2e302781fc5a573acd2c14fcfb2aa85c6ce29 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=12, curl_handle=0x564c6f81eb70 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4e96f52900 20 run: stack=0x564c6f8210c0 is io blocked 2026-03-20T11:46:23.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.082+0000 7f4df2ffd640 20 link_request req_data=0x564c6f81f660 req_data->id=12, curl_handle=0x564c6f81eb70 2026-03-20T11:46:23.081 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:23.082+0000 7fd002252640 -1 req 4151458275832474261 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:23.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.085+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f90adb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.085+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f90adb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.085+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f90adb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.085+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f90adb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.085+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f914120:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.085+0000 7f4e96f52900 15 stack 0x564c6f90d800 end 2026-03-20T11:46:23.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.085+0000 7f4e96f52900 20 run: stack=0x564c6f90d800 is done 2026-03-20T11:46:23.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.085+0000 7f4e96f52900 20 cr:s=0x564c6f919040:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.085+0000 7f4e96f52900 20 collect(): s=0x564c6f919040 stack=0x564c6f90d800 is complete 2026-03-20T11:46:23.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.085+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.085+0000 7f4e96f52900 10 trimming bilogs for bucket=:dipepu-23[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28]) markers=[,,,,,,,,00000000003.16.7,,], shards=11 2026-03-20T11:46:23.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.085+0000 7f4e96f52900 20 cr:s=0x564c6f919040:op=0x564c6f91c3c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.085+0000 7f4e96f52900 10 trimming bilog shard 8 of :dipepu-23[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28]) at marker 00000000003.16.7 2026-03-20T11:46:23.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.085+0000 7f4e96f52900 20 run: stack=0x564c6f919040 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.085+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f90adb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.085+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f90adb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.085+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28.8 2026-03-20T11:46:23.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.085+0000 7f4e96f52900 20 run: stack=0x564c6f90d800 is io blocked 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f90e450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f90e450:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f917ae0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 15 stack 0x564c6f907e20 end 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is done 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f909ae0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 collect(): s=0x564c6f93f450 stack=0x564c6f907e20 is complete 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f909ae0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 15 stack 0x564c6f93f450 end 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 stack->operate() returned ret=-22 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is done 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f909ae0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 4 starting trim on bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f9563d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f9563d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 enqueued request req=0x564c6f81c970 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 req: 0x564c6f81c970 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is io blocked 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e4c7f0640 20 dequeued request req=0x564c6f81c970 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e4c7f0640 20 RGWWQ: empty 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e4c7f0640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 : miss 2026-03-20T11:46:23.083 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e4c7f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e4c7f0640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e4c7f0640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 info.flags=0x17 2026-03-20T11:46:23.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e4c7f0640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 to cache LRU end 2026-03-20T11:46:23.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e4c7f0640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e4c7f0640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e4c7f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:46:23.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e4c7f0640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-19 : miss 2026-03-20T11:46:23.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e4c7f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e4c7f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e4c7f0640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-19 info.flags=0x0 2026-03-20T11:46:23.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e4c7f0640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-19 to cache LRU end 2026-03-20T11:46:23.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e4c7f0640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-19 : miss 2026-03-20T11:46:23.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e4c7f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f90adb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f90adb0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 15 stack 0x564c6f90d800 end 2026-03-20T11:46:23.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 run: stack=0x564c6f90d800 is done 2026-03-20T11:46:23.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 cr:s=0x564c6f919040:op=0x564c6f91c3c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 cr:s=0x564c6f919040:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 trim: trimmed bucket instance dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:23.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 15 stack 0x564c6f919040 end 2026-03-20T11:46:23.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 run: stack=0x564c6f919040 is done 2026-03-20T11:46:23.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 4 starting trim on bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f90adb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f90adb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 enqueued request req=0x564c6f94ce80 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 req: 0x564c6f94ce80 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e96f52900 20 run: stack=0x564c6f90d800 is io blocked 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e4bfef640 20 dequeued request req=0x564c6f94ce80 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e4bfef640 20 RGWWQ: empty 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e4bfef640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 : miss 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.086+0000 7f4e4bfef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4bfef640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4bfef640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 info.flags=0x17 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4bfef640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 to cache LRU end 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4bfef640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4bfef640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4bfef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4bfef640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-4 : miss 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4bfef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4c7f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4c7f0640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-19 info.flags=0x0 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4c7f0640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-19 to cache LRU end 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4c7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24): adding source pipe: {s={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4c7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24): adding dest pipe: {s={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4c7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4c7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4c7f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f9563d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f9563d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f9563d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f9563d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f909ae0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f914100:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f909ae0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f90e450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f90e450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = e478a87df6a18dcaa3c32665e17cad82452b6381ae6da3f4e57d7d798c85200e 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:e478a87df6a18dcaa3c32665e17cad82452b6381ae6da3f4e57d7d798c85200e 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = a3dff676261ecb055709a0ed7ef5bce2e7ebc0e3d831f8da7923d7ceff2795f5 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a3dff676261ecb055709a0ed7ef5bce2e7ebc0e3d831f8da7923d7ceff2795f5 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=13, curl_handle=0x564c6f823420 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4bfef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4bfef640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-4 info.flags=0x0 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4bfef640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-4 to cache LRU end 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4bfef640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-4 : miss 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4bfef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is io blocked 2026-03-20T11:46:23.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4df2ffd640 20 link_request req_data=0x564c6f918e60 req_data->id=13, curl_handle=0x564c6f823420 2026-03-20T11:46:23.087 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:23.087+0000 7fcfdd208640 -1 req 8035032859012896356 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4bfef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4bfef640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-4 info.flags=0x0 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4bfef640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-4 to cache LRU end 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4bfef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5): adding source pipe: {s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4bfef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5): adding dest pipe: {s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4bfef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4bfef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e4bfef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f90adb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f90adb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f90adb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f90adb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f948f20:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 run: stack=0x564c6f90d800 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f9557b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f9557b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.088 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = d8567393421d7c5eb2519ca8c86357a3cfdc9bb15c4741ffdfe9dfa670e68661 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:d8567393421d7c5eb2519ca8c86357a3cfdc9bb15c4741ffdfe9dfa670e68661 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 9a3fc23c258ba6989832fdca296f08e1cdbb2fdbaf0603e5715b8add4ad99f65 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=9a3fc23c258ba6989832fdca296f08e1cdbb2fdbaf0603e5715b8add4ad99f65 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.087+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=14, curl_handle=0x564c6f911bf0 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.088+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is io blocked 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.088+0000 7f4df2ffd640 20 link_request req_data=0x564c6f912300 req_data->id=14, curl_handle=0x564c6f911bf0 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f9524f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f9524f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f9524f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f9524f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f9113b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 15 stack 0x564c6f821700 end 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is done 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 collect(): s=0x564c6f907840 stack=0x564c6f821700 is complete 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 10 trimming bilogs for bucket=:dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) markers=[00000000004.46.6,00000000002.37.1,00000000004.60.6,00000000002.19.1,00000000004.18.6,00000000002.57.1,00000000002.32.1,00000000004.68.6,00000000002.43.1,00000000002.85.1,00000000002.30.1], shards=11 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f91c3c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 10 trimming bilog shard 0 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.46.6 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 10 trimming bilog shard 1 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.37.1 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 10 trimming bilog shard 2 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.60.6 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 10 trimming bilog shard 3 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.19.1 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 10 trimming bilog shard 4 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.18.6 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 10 trimming bilog shard 5 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.57.1 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 10 trimming bilog shard 6 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.32.1 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 10 trimming bilog shard 7 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.68.6 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 10 trimming bilog shard 8 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.43.1 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 10 trimming bilog shard 9 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.85.1 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 10 trimming bilog shard 10 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.30.1 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 run: stack=0x564c6f907840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f90adb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f917ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f821e70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f948720:op=0x564c6f49f9b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f9155e0:op=0x564c6f919dc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f947550:op=0x564c6f91b980:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f914720:op=0x564c6f9519b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f947990:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f94b640:op=0x564c6f81ccf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f946fd0:op=0x564c6f90f9b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f4a4760:op=0x564c6f4a0dd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f90adb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.0 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is io blocked 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f917ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.1 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 run: stack=0x564c6f94b500 is io blocked 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f821e70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.2 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 run: stack=0x564c6f953ef0 is io blocked 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f948720:op=0x564c6f49f9b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.3 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 run: stack=0x564c6f948720 is io blocked 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f9155e0:op=0x564c6f919dc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.4 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 run: stack=0x564c6f9155e0 is io blocked 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f947550:op=0x564c6f91b980:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.5 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 run: stack=0x564c6f947550 is io blocked 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f914720:op=0x564c6f9519b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.6 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 run: stack=0x564c6f914720 is io blocked 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f947990:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.7 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 run: stack=0x564c6f94c000 is io blocked 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f94b640:op=0x564c6f81ccf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.8 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 run: stack=0x564c6f94b640 is io blocked 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f946fd0:op=0x564c6f90f9b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.9 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 run: stack=0x564c6f946fd0 is io blocked 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 cr:s=0x564c6f4a4760:op=0x564c6f4a0dd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.10 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.089+0000 7f4e96f52900 20 run: stack=0x564c6f4a4760 is io blocked 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f917ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f917ae0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 15 stack 0x564c6f94b500 end 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 run: stack=0x564c6f94b500 is done 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f91c3c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 run: stack=0x564c6f907840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 cr:s=0x564c6f948720:op=0x564c6f49f9b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 cr:s=0x564c6f948720:op=0x564c6f49f9b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 15 stack 0x564c6f948720 end 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 run: stack=0x564c6f948720 is done 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f91c3c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 run: stack=0x564c6f907840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 cr:s=0x564c6f947550:op=0x564c6f91b980:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 cr:s=0x564c6f947550:op=0x564c6f91b980:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 15 stack 0x564c6f947550 end 2026-03-20T11:46:23.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 run: stack=0x564c6f947550 is done 2026-03-20T11:46:23.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f91c3c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 run: stack=0x564c6f907840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 cr:s=0x564c6f94b640:op=0x564c6f81ccf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 cr:s=0x564c6f94b640:op=0x564c6f81ccf0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 15 stack 0x564c6f94b640 end 2026-03-20T11:46:23.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 run: stack=0x564c6f94b640 is done 2026-03-20T11:46:23.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f90adb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f90adb0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 15 stack 0x564c6f821700 end 2026-03-20T11:46:23.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is done 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f91c3c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f91c3c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 run: stack=0x564c6f907840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 cr:s=0x564c6f4a4760:op=0x564c6f4a0dd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 cr:s=0x564c6f4a4760:op=0x564c6f4a0dd0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 15 stack 0x564c6f4a4760 end 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 run: stack=0x564c6f4a4760 is done 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f91c3c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 run: stack=0x564c6f907840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f821e70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f821e70:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 15 stack 0x564c6f953ef0 end 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 run: stack=0x564c6f953ef0 is done 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f91c3c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 run: stack=0x564c6f907840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f947990:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f947990:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 15 stack 0x564c6f94c000 end 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 run: stack=0x564c6f94c000 is done 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f91c3c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.091+0000 7f4e96f52900 20 run: stack=0x564c6f907840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 cr:s=0x564c6f914720:op=0x564c6f9519b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 cr:s=0x564c6f914720:op=0x564c6f9519b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 15 stack 0x564c6f914720 end 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 run: stack=0x564c6f914720 is done 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f91c3c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 run: stack=0x564c6f907840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 cr:s=0x564c6f9155e0:op=0x564c6f919dc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 cr:s=0x564c6f9155e0:op=0x564c6f919dc0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 15 stack 0x564c6f9155e0 end 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 run: stack=0x564c6f9155e0 is done 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f91c3c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 run: stack=0x564c6f907840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 cr:s=0x564c6f946fd0:op=0x564c6f90f9b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 cr:s=0x564c6f946fd0:op=0x564c6f90f9b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 15 stack 0x564c6f946fd0 end 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 run: stack=0x564c6f946fd0 is done 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f91c3c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 cr:s=0x564c6f907840:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 trim: trimmed bucket instance dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 15 stack 0x564c6f907840 end 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 run: stack=0x564c6f907840 is done 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 4 starting trim on bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f90adb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f90adb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 enqueued request req=0x564c6f90ec10 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 req: 0x564c6f90ec10 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e96f52900 20 run: stack=0x564c6f94c000 is io blocked 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e4b7ee640 20 dequeued request req=0x564c6f90ec10 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e4b7ee640 20 RGWWQ: empty 2026-03-20T11:46:23.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e4b7ee640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 : miss 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e4b7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e4b7ee640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e4b7ee640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 info.flags=0x17 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e4b7ee640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 to cache LRU end 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e4b7ee640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e4b7ee640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e4b7ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e4b7ee640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-53 : miss 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.092+0000 7f4e4b7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e4b7ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e4b7ee640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-53 info.flags=0x0 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e4b7ee640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-53 to cache LRU end 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e4b7ee640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-53 : miss 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e4b7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e4b7ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e4b7ee640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-53 info.flags=0x0 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e4b7ee640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-53 to cache LRU end 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e4b7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13): adding source pipe: {s={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e4b7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13): adding dest pipe: {s={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e4b7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e4b7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e4b7ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f90adb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f90adb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f90adb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f90adb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f4a5950:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 20 run: stack=0x564c6f94c000 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f91c3c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f91c3c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = c3bc8feec39d259cba60eb7a269dea8c8366b9121358408c8a5fcc49e5ec710a 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.093 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.094 INFO:teuthology.orchestra.run.vm02.stderr:c3bc8feec39d259cba60eb7a269dea8c8366b9121358408c8a5fcc49e5ec710a 2026-03-20T11:46:23.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = d1cbbdadbb6d705940a4c48d069fb4bdf07dd4f4f1cd178dc7ce97f3b606da5c 2026-03-20T11:46:23.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=d1cbbdadbb6d705940a4c48d069fb4bdf07dd4f4f1cd178dc7ce97f3b606da5c 2026-03-20T11:46:23.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=15, curl_handle=0x564c6f915d30 2026-03-20T11:46:23.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4e96f52900 20 run: stack=0x564c6f953ef0 is io blocked 2026-03-20T11:46:23.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.093+0000 7f4df2ffd640 20 link_request req_data=0x564c6f954490 req_data->id=15, curl_handle=0x564c6f915d30 2026-03-20T11:46:23.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f910820:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f910820:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:23.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f821840:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e96f52900 15 stack 0x564c6f8210c0 end 2026-03-20T11:46:23.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e96f52900 20 run: stack=0x564c6f8210c0 is done 2026-03-20T11:46:23.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e96f52900 20 collect(): s=0x564c6f8132d0 stack=0x564c6f8210c0 is complete 2026-03-20T11:46:23.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e96f52900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:23.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e96f52900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:23.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:23.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e96f52900 15 stack 0x564c6f8132d0 end 2026-03-20T11:46:23.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e96f52900 20 stack->operate() returned ret=-22 2026-03-20T11:46:23.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is done 2026-03-20T11:46:23.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e96f52900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:23.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e96f52900 4 starting trim on bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:46:23.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f9113b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f9113b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e96f52900 20 enqueued request req=0x564c6f94c560 2026-03-20T11:46:23.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e96f52900 20 req: 0x564c6f94c560 2026-03-20T11:46:23.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is io blocked 2026-03-20T11:46:23.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e4afed640 20 dequeued request req=0x564c6f94c560 2026-03-20T11:46:23.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e4afed640 20 RGWWQ: empty 2026-03-20T11:46:23.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e4afed640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 : miss 2026-03-20T11:46:23.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.124+0000 7f4e4afed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.125+0000 7f4e4afed640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.125+0000 7f4e4afed640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 info.flags=0x17 2026-03-20T11:46:23.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.125+0000 7f4e4afed640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 to cache LRU end 2026-03-20T11:46:23.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.125+0000 7f4e4afed640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.125+0000 7f4e4afed640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.125+0000 7f4e4afed640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:46:23.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.125+0000 7f4e4afed640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-14 : miss 2026-03-20T11:46:23.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.125+0000 7f4e4afed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.125+0000 7f4e4afed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.125+0000 7f4e4afed640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-14 info.flags=0x0 2026-03-20T11:46:23.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.125+0000 7f4e4afed640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-14 to cache LRU end 2026-03-20T11:46:23.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.125+0000 7f4e4afed640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-14 : miss 2026-03-20T11:46:23.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.125+0000 7f4e4afed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e4afed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e4afed640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-14 info.flags=0x0 2026-03-20T11:46:23.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e4afed640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-14 to cache LRU end 2026-03-20T11:46:23.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e4afed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19): adding source pipe: {s={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e4afed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19): adding dest pipe: {s={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e4afed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e4afed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e4afed640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f9113b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f9113b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f9113b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f9113b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f9113b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f952430:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f952430:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 8fef127fc3383e44b04b7ed0c8abaae78d6712c80f766632ffaf9ff7b2c43052 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:8fef127fc3383e44b04b7ed0c8abaae78d6712c80f766632ffaf9ff7b2c43052 2026-03-20T11:46:23.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.124 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.124 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.124 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.124 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 37c00dd668dd38602025f3651d439d4810b0097d4acd833284c1b04d7f05eee2 2026-03-20T11:46:23.124 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=37c00dd668dd38602025f3651d439d4810b0097d4acd833284c1b04d7f05eee2 2026-03-20T11:46:23.124 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.124 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.124 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.124 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=16, curl_handle=0x564c6f81eb70 2026-03-20T11:46:23.124 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4e96f52900 20 run: stack=0x564c6f8210c0 is io blocked 2026-03-20T11:46:23.124 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.126+0000 7f4df2ffd640 20 link_request req_data=0x564c6f81f660 req_data->id=16, curl_handle=0x564c6f81eb70 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f9557b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f9557b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f948f20:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e96f52900 15 stack 0x564c6f94bd80 end 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is done 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e96f52900 20 collect(): s=0x564c6f90d800 stack=0x564c6f94bd80 is complete 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e96f52900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e96f52900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e96f52900 15 stack 0x564c6f90d800 end 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e96f52900 20 stack->operate() returned ret=-22 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e96f52900 20 run: stack=0x564c6f90d800 is done 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e96f52900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e96f52900 4 starting trim on bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f947990:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f947990:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e96f52900 20 enqueued request req=0x564c6f94ce80 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e96f52900 20 req: 0x564c6f94ce80 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e96f52900 20 run: stack=0x564c6f90d800 is io blocked 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e4a7ec640 20 dequeued request req=0x564c6f94ce80 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e4a7ec640 20 RGWWQ: empty 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e4a7ec640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 : miss 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e4a7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e4a7ec640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e4a7ec640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 info.flags=0x17 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e4a7ec640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 to cache LRU end 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e4a7ec640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e4a7ec640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e4a7ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e4a7ec640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-36 : miss 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e4a7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e4a7ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e4a7ec640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-36 info.flags=0x0 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e4a7ec640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-36 to cache LRU end 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e4a7ec640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-36 : miss 2026-03-20T11:46:23.126 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.129+0000 7f4e4a7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f90e450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e4a7ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e4a7ec640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-36 info.flags=0x0 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e4a7ec640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-36 to cache LRU end 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e4a7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41): adding source pipe: {s={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e4a7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41): adding dest pipe: {s={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e4a7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e4a7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e4a7ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f90e450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f947990:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f90e450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f947990:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f90e450:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f947990:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f914100:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 15 stack 0x564c6f907e20 end 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is done 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f947990:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f909ae0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 20 collect(): s=0x564c6f93f450 stack=0x564c6f907e20 is complete 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 10 trimming bilogs for bucket=:dipepu-19[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24]) markers=[,,,,,,,,00000000003.27.6,,], shards=11 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f4a0dd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 10 trimming bilog shard 8 of :dipepu-19[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24]) at marker 00000000003.27.6 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f91b360:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 20 run: stack=0x564c6f90d800 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f90adb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f9563d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f90adb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24.8 2026-03-20T11:46:23.127 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is io blocked 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f9563d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.130+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.131+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.131+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.131+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.131+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = d35caa07cab65a078bc7b50b40fb7ef1ad7fc59d35aecb0bb407d6c476c2f6aa 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.131+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:d35caa07cab65a078bc7b50b40fb7ef1ad7fc59d35aecb0bb407d6c476c2f6aa 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.131+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.131+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.131+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.131+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.131+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 95903682e6ed679d620ff8c8aa5162266c2940e37754cb3cd9151be49e0ed114 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.131+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=95903682e6ed679d620ff8c8aa5162266c2940e37754cb3cd9151be49e0ed114 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.131+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.131+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.131+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.131+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=17, curl_handle=0x564c6f823420 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.131+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is io blocked 2026-03-20T11:46:23.128 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.131+0000 7f4df2ffd640 20 link_request req_data=0x564c6f918e60 req_data->id=17, curl_handle=0x564c6f823420 2026-03-20T11:46:23.129 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.132+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f90adb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.129 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.132+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f90adb0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.129 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.132+0000 7f4e96f52900 15 stack 0x564c6f94bd80 end 2026-03-20T11:46:23.129 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.132+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.129 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.132+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is done 2026-03-20T11:46:23.129 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.132+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f4a0dd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.129 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.132+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f909ae0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.129 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.132+0000 7f4e96f52900 20 trim: trimmed bucket instance dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:46:23.129 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.132+0000 7f4e96f52900 15 stack 0x564c6f93f450 end 2026-03-20T11:46:23.129 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.132+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is done 2026-03-20T11:46:23.129 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.132+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.129 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.132+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.129 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.132+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f94d810:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.129 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.132+0000 7f4e96f52900 4 starting trim on bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:46:23.129 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.132+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f947990:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.129 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.132+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f947990:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.129 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.132+0000 7f4e96f52900 20 enqueued request req=0x564c6f81c970 2026-03-20T11:46:23.129 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.132+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.129 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.132+0000 7f4e96f52900 20 req: 0x564c6f81c970 2026-03-20T11:46:23.129 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.132+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is io blocked 2026-03-20T11:46:23.129 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.132+0000 7f4e49feb640 20 dequeued request req=0x564c6f81c970 2026-03-20T11:46:23.129 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.132+0000 7f4e49feb640 20 RGWWQ: empty 2026-03-20T11:46:23.129 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.132+0000 7f4e49feb640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 : miss 2026-03-20T11:46:23.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.132+0000 7f4e49feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e49feb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e49feb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 info.flags=0x17 2026-03-20T11:46:23.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e49feb640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 to cache LRU end 2026-03-20T11:46:23.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e49feb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e49feb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e49feb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:46:23.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e49feb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-22 : miss 2026-03-20T11:46:23.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e49feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e49feb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e49feb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-22 info.flags=0x0 2026-03-20T11:46:23.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e49feb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-22 to cache LRU end 2026-03-20T11:46:23.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e49feb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-22 : miss 2026-03-20T11:46:23.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e49feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e49feb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e49feb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-22 info.flags=0x0 2026-03-20T11:46:23.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e49feb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-22 to cache LRU end 2026-03-20T11:46:23.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e49feb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27): adding source pipe: {s={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e49feb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27): adding dest pipe: {s={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e49feb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e49feb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e49feb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:46:23.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f947990:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f947990:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f947990:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f947990:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f94d810:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f90e430:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f94d810:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 49a548ee98fdea1fb7d7f7b99fdc7535070bee872bfff6b50a10496c9fe72c06 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:49a548ee98fdea1fb7d7f7b99fdc7535070bee872bfff6b50a10496c9fe72c06 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 5dc67a676b4c5a73676dece6eae23bbffa926a13a9d9a191a484b0d2595017bd 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5dc67a676b4c5a73676dece6eae23bbffa926a13a9d9a191a484b0d2595017bd 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=18, curl_handle=0x564c6f911bf0 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is io blocked 2026-03-20T11:46:23.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.133+0000 7f4df2ffd640 20 link_request req_data=0x564c6f912300 req_data->id=18, curl_handle=0x564c6f911bf0 2026-03-20T11:46:23.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f91c3c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f91c3c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f91c3c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f91c3c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f4a5950:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 15 stack 0x564c6f953ef0 end 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 run: stack=0x564c6f953ef0 is done 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 collect(): s=0x564c6f94c000 stack=0x564c6f953ef0 is complete 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 10 trimming bilogs for bucket=:dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) markers=[00000000004.65.1,00000000002.83.1,00000000004.51.1,00000000002.21.1,00000000004.74.1,00000000002.72.1,00000000002.84.1,00000000004.73.1,00000000002.22.1,00000000002.50.1,00000000002.35.1], shards=11 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f4a5950:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 10 trimming bilog shard 0 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.65.1 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 10 trimming bilog shard 1 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.83.1 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 10 trimming bilog shard 2 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.51.1 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 10 trimming bilog shard 3 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.21.1 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 10 trimming bilog shard 4 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.74.1 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 10 trimming bilog shard 5 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.72.1 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 10 trimming bilog shard 6 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.84.1 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 10 trimming bilog shard 7 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.73.1 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 10 trimming bilog shard 8 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.22.1 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 10 trimming bilog shard 9 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.50.1 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 10 trimming bilog shard 10 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.35.1 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 run: stack=0x564c6f94c000 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f90b800:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f4a4760:op=0x564c6f81ccf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f94b640:op=0x564c6f49f9b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f962ea0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f948720:op=0x564c6f9621b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f960640:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f9613e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f9578d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f4a74d0:op=0x564c6f821840:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f4a1900:op=0x564c6f966920:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f967120:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f90b800:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.0 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 run: stack=0x564c6f953ef0 is io blocked 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f4a4760:op=0x564c6f81ccf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.1 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 run: stack=0x564c6f4a4760 is io blocked 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f94b640:op=0x564c6f49f9b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.2 2026-03-20T11:46:23.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 run: stack=0x564c6f94b640 is io blocked 2026-03-20T11:46:23.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f962ea0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.3 2026-03-20T11:46:23.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is io blocked 2026-03-20T11:46:23.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f948720:op=0x564c6f9621b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.4 2026-03-20T11:46:23.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 run: stack=0x564c6f948720 is io blocked 2026-03-20T11:46:23.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f960640:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.5 2026-03-20T11:46:23.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 run: stack=0x564c6f94b500 is io blocked 2026-03-20T11:46:23.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f9613e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.6 2026-03-20T11:46:23.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is io blocked 2026-03-20T11:46:23.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f9578d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.7 2026-03-20T11:46:23.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is io blocked 2026-03-20T11:46:23.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f4a74d0:op=0x564c6f821840:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.8 2026-03-20T11:46:23.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 run: stack=0x564c6f4a74d0 is io blocked 2026-03-20T11:46:23.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f4a1900:op=0x564c6f966920:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.9 2026-03-20T11:46:23.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 run: stack=0x564c6f4a1900 is io blocked 2026-03-20T11:46:23.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f967120:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.10 2026-03-20T11:46:23.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.136+0000 7f4e96f52900 20 run: stack=0x564c6f9106c0 is io blocked 2026-03-20T11:46:23.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 cr:s=0x564c6f4a1900:op=0x564c6f966920:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 cr:s=0x564c6f4a1900:op=0x564c6f966920:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 15 stack 0x564c6f4a1900 end 2026-03-20T11:46:23.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 run: stack=0x564c6f4a1900 is done 2026-03-20T11:46:23.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f4a5950:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 run: stack=0x564c6f94c000 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 cr:s=0x564c6f4a4760:op=0x564c6f81ccf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 cr:s=0x564c6f4a4760:op=0x564c6f81ccf0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 15 stack 0x564c6f4a4760 end 2026-03-20T11:46:23.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 run: stack=0x564c6f4a4760 is done 2026-03-20T11:46:23.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f4a5950:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 run: stack=0x564c6f94c000 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f962ea0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f962ea0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 15 stack 0x564c6f821700 end 2026-03-20T11:46:23.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is done 2026-03-20T11:46:23.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f4a5950:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.135 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 run: stack=0x564c6f94c000 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f90b800:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f90b800:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 15 stack 0x564c6f953ef0 end 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 run: stack=0x564c6f953ef0 is done 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f4a5950:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 run: stack=0x564c6f94c000 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 cr:s=0x564c6f94b640:op=0x564c6f49f9b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 cr:s=0x564c6f94b640:op=0x564c6f49f9b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 15 stack 0x564c6f94b640 end 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 run: stack=0x564c6f94b640 is done 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f967120:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f967120:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 15 stack 0x564c6f9106c0 end 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 run: stack=0x564c6f9106c0 is done 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f960640:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f960640:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 15 stack 0x564c6f94b500 end 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 run: stack=0x564c6f94b500 is done 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f4a5950:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f4a5950:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f4a5950:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 run: stack=0x564c6f94c000 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f9578d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f9578d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.138+0000 7f4e96f52900 15 stack 0x564c6f4a7b60 end 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is done 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f4a5950:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 run: stack=0x564c6f94c000 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f9613e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f9613e0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 15 stack 0x564c6f4a77d0 end 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is done 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f4a5950:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 run: stack=0x564c6f94c000 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.136 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 cr:s=0x564c6f948720:op=0x564c6f9621b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 cr:s=0x564c6f948720:op=0x564c6f9621b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 15 stack 0x564c6f948720 end 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 run: stack=0x564c6f948720 is done 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f4a5950:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 run: stack=0x564c6f94c000 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 cr:s=0x564c6f4a74d0:op=0x564c6f821840:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 cr:s=0x564c6f4a74d0:op=0x564c6f821840:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 15 stack 0x564c6f4a74d0 end 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 run: stack=0x564c6f4a74d0 is done 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f4a5950:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 trim: trimmed bucket instance dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 15 stack 0x564c6f94c000 end 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 run: stack=0x564c6f94c000 is done 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 4 starting trim on bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f4a5950:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f4a5950:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 enqueued request req=0x564c6f4a0b10 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 req: 0x564c6f4a0b10 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is io blocked 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e497ea640 20 dequeued request req=0x564c6f4a0b10 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e497ea640 20 RGWWQ: empty 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e497ea640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 : miss 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e497ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e497ea640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e497ea640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 info.flags=0x17 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e497ea640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 to cache LRU end 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e497ea640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e497ea640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e497ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e497ea640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-25 : miss 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.139+0000 7f4e497ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e497ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e497ea640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-25 info.flags=0x0 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e497ea640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-25 to cache LRU end 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e497ea640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-25 : miss 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e497ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e497ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e497ea640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-25 info.flags=0x0 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e497ea640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-25 to cache LRU end 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e497ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30): adding source pipe: {s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e497ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30): adding dest pipe: {s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e497ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e497ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e497ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f4a5950:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f4a5950:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f4a5950:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f4a5950:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f4a5950:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f914100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f914100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 36c505d4f06d30c2195fd6553c74916bebfee93b1e453b5f3ca3307134a8223b 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:36c505d4f06d30c2195fd6553c74916bebfee93b1e453b5f3ca3307134a8223b 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = b56954c708c4f100e3ccd9cb8e237838cb39ef68b804cbb6b8d5dbd2f161530d 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b56954c708c4f100e3ccd9cb8e237838cb39ef68b804cbb6b8d5dbd2f161530d 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=19, curl_handle=0x564c6f915d30 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4e96f52900 20 run: stack=0x564c6f9106c0 is io blocked 2026-03-20T11:46:23.138 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.140+0000 7f4df2ffd640 20 link_request req_data=0x564c6f954490 req_data->id=19, curl_handle=0x564c6f915d30 2026-03-20T11:46:23.139 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:23.140+0000 7fd06cb27640 -1 req 7914144792835004154 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:23.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.169+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f952430:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.169+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f952430:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.169+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f952430:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.169+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f952430:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.169+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f9113b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.169+0000 7f4e96f52900 15 stack 0x564c6f8210c0 end 2026-03-20T11:46:23.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.169+0000 7f4e96f52900 20 run: stack=0x564c6f8210c0 is done 2026-03-20T11:46:23.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.169+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.169+0000 7f4e96f52900 20 collect(): s=0x564c6f8132d0 stack=0x564c6f8210c0 is complete 2026-03-20T11:46:23.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.169+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.169+0000 7f4e96f52900 10 trimming bilogs for bucket=:dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) markers=[,,,,,00000000002.32.11,,,00000000003.22.6,00000000011.52.6,], shards=11 2026-03-20T11:46:23.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.169+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f9113b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.169+0000 7f4e96f52900 10 trimming bilog shard 5 of :dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) at marker 00000000002.32.11 2026-03-20T11:46:23.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.169+0000 7f4e96f52900 10 trimming bilog shard 8 of :dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) at marker 00000000003.22.6 2026-03-20T11:46:23.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.169+0000 7f4e96f52900 10 trimming bilog shard 9 of :dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) at marker 00000000011.52.6 2026-03-20T11:46:23.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.169+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.169+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f94f7e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.169+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f81ccf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.169+0000 7f4e96f52900 20 cr:s=0x564c6f94b640:op=0x564c6f963480:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.169+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f94f7e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.169+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19.5 2026-03-20T11:46:23.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.169+0000 7f4e96f52900 20 run: stack=0x564c6f8210c0 is io blocked 2026-03-20T11:46:23.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.169+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f81ccf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.169+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19.8 2026-03-20T11:46:23.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.169+0000 7f4e96f52900 20 run: stack=0x564c6f94b500 is io blocked 2026-03-20T11:46:23.167 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.169+0000 7f4e96f52900 20 cr:s=0x564c6f94b640:op=0x564c6f963480:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.167 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.169+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19.9 2026-03-20T11:46:23.167 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.169+0000 7f4e96f52900 20 run: stack=0x564c6f94b640 is io blocked 2026-03-20T11:46:23.168 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.171+0000 7f4e96f52900 20 cr:s=0x564c6f94b640:op=0x564c6f963480:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.168 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.171+0000 7f4e96f52900 20 cr:s=0x564c6f94b640:op=0x564c6f963480:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.168 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.171+0000 7f4e96f52900 15 stack 0x564c6f94b640 end 2026-03-20T11:46:23.168 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.171+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.168 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.171+0000 7f4e96f52900 20 run: stack=0x564c6f94b640 is done 2026-03-20T11:46:23.168 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.171+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f9113b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.168 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.171+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.168 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.171+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f94f7e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.168 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.171+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f94f7e0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.168 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.171+0000 7f4e96f52900 15 stack 0x564c6f8210c0 end 2026-03-20T11:46:23.168 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.171+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.168 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.171+0000 7f4e96f52900 20 run: stack=0x564c6f8210c0 is done 2026-03-20T11:46:23.168 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.171+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f9113b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.168 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.171+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.168 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.171+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f81ccf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.171+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f81ccf0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.171+0000 7f4e96f52900 15 stack 0x564c6f94b500 end 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.171+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.171+0000 7f4e96f52900 20 run: stack=0x564c6f94b500 is done 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.171+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f9113b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.171+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.171+0000 7f4e96f52900 20 trim: trimmed bucket instance dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.171+0000 7f4e96f52900 15 stack 0x564c6f8132d0 end 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.172+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is done 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.172+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.172+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.172+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.172+0000 7f4e96f52900 4 starting trim on bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.172+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f94f7e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.172+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f94f7e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.172+0000 7f4e96f52900 20 enqueued request req=0x564c6f90ec10 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.172+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.172+0000 7f4e96f52900 20 req: 0x564c6f90ec10 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.172+0000 7f4e48fe9640 20 dequeued request req=0x564c6f90ec10 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.172+0000 7f4e48fe9640 20 RGWWQ: empty 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.172+0000 7f4e48fe9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 : miss 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.172+0000 7f4e48fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.172+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is io blocked 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.172+0000 7f4e48fe9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.172+0000 7f4e48fe9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 info.flags=0x17 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.172+0000 7f4e48fe9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 to cache LRU end 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.172+0000 7f4e48fe9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.172+0000 7f4e48fe9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.172+0000 7f4e48fe9640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.172+0000 7f4e48fe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.172+0000 7f4e48fe9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-35 : miss 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.172+0000 7f4e48fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e48fe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.169 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e48fe9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-35 info.flags=0x0 2026-03-20T11:46:23.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e48fe9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-35 to cache LRU end 2026-03-20T11:46:23.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e48fe9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-35 : miss 2026-03-20T11:46:23.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e48fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f9563d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f9563d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f9563d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f9563d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f91b360:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 15 stack 0x564c6f907e20 end 2026-03-20T11:46:23.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is done 2026-03-20T11:46:23.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 collect(): s=0x564c6f90d800 stack=0x564c6f907e20 is complete 2026-03-20T11:46:23.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 10 trimming bilogs for bucket=:dipepu-36[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41]) markers=[,,,,,,,,,,00000000001.34.6], shards=11 2026-03-20T11:46:23.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f9113b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 10 trimming bilog shard 10 of :dipepu-36[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41]) at marker 00000000001.34.6 2026-03-20T11:46:23.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 run: stack=0x564c6f90d800 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f4a02e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f4a02e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41.10 2026-03-20T11:46:23.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e48fe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e48fe9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-35 info.flags=0x0 2026-03-20T11:46:23.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e48fe9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-35 to cache LRU end 2026-03-20T11:46:23.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e48fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40): adding source pipe: {s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e48fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40): adding dest pipe: {s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e48fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e48fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.170 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e48fe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is io blocked 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f94f7e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f94f7e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f94f7e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f94f7e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f95b440:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f95eeb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f95eeb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 1a1bb4cc1235bc322f8344ccd54fdeb9acad11bd305c3038b607ee957fcd5e23 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.171 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.172 INFO:teuthology.orchestra.run.vm02.stderr:1a1bb4cc1235bc322f8344ccd54fdeb9acad11bd305c3038b607ee957fcd5e23 2026-03-20T11:46:23.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 9eca4a8f2e70bb0f403dfd213f45ccf6b6a3ea4218f5ff4efd4985396687975f 2026-03-20T11:46:23.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=9eca4a8f2e70bb0f403dfd213f45ccf6b6a3ea4218f5ff4efd4985396687975f 2026-03-20T11:46:23.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=20, curl_handle=0x564c6f823420 2026-03-20T11:46:23.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4e96f52900 20 run: stack=0x564c6f94b500 is io blocked 2026-03-20T11:46:23.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.173+0000 7f4df2ffd640 20 link_request req_data=0x564c6f918e60 req_data->id=20, curl_handle=0x564c6f823420 2026-03-20T11:46:23.172 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:23.173+0000 7fcfeaa23640 -1 req 880590126450693179 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:23.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f4a02e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f4a02e0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e96f52900 15 stack 0x564c6f907e20 end 2026-03-20T11:46:23.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is done 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f9113b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e96f52900 20 trim: trimmed bucket instance dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e96f52900 15 stack 0x564c6f90d800 end 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e96f52900 20 run: stack=0x564c6f90d800 is done 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e96f52900 4 starting trim on bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f963900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f963900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e96f52900 20 enqueued request req=0x564c6f4a34a0 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e96f52900 20 req: 0x564c6f4a34a0 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e487e8640 20 dequeued request req=0x564c6f4a34a0 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e487e8640 20 RGWWQ: empty 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e487e8640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 : miss 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e487e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e96f52900 20 run: stack=0x564c6f90d800 is io blocked 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e487e8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e487e8640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 info.flags=0x17 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e487e8640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 to cache LRU end 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e487e8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e487e8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e487e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e487e8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-26 : miss 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.175+0000 7f4e487e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e487e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e487e8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-26 info.flags=0x0 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e487e8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-26 to cache LRU end 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e487e8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-26 : miss 2026-03-20T11:46:23.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e487e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f90e430:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 15 stack 0x564c6f94bd80 end 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is done 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f94d810:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 20 collect(): s=0x564c6f93f450 stack=0x564c6f94bd80 is complete 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 10 trimming bilogs for bucket=:dipepu-22[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27]) markers=[,,,,,,,,00000000006.25.9,,], shards=11 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f9113b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 10 trimming bilog shard 8 of :dipepu-22[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27]) at marker 00000000006.25.9 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f4a02e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f4a02e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e487e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e487e8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-26 info.flags=0x0 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e487e8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-26 to cache LRU end 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e487e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31): adding source pipe: {s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e487e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31): adding dest pipe: {s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e487e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e487e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e487e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27.8 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is io blocked 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f963900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f963900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f963900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f963900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f91b520:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 20 run: stack=0x564c6f90d800 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = e664a42785138cd6c431530a853530db46d4684e4c4783a70739fceba59a1db3 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:e664a42785138cd6c431530a853530db46d4684e4c4783a70739fceba59a1db3 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.176+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.177+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = cf8da6ca7f038a111a7ce63ef99b732b87c6d523dd274216ec0cc5105b499b41 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.177+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=cf8da6ca7f038a111a7ce63ef99b732b87c6d523dd274216ec0cc5105b499b41 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.177+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.177+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.177+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.177+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=21, curl_handle=0x564c6f911bf0 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.177+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is io blocked 2026-03-20T11:46:23.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.177+0000 7f4df2ffd640 20 link_request req_data=0x564c6f912300 req_data->id=21, curl_handle=0x564c6f911bf0 2026-03-20T11:46:23.175 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:23.177+0000 7fcff0a2f640 -1 req 4203261194450609120 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:23.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f4a02e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f4a02e0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e96f52900 15 stack 0x564c6f94bd80 end 2026-03-20T11:46:23.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is done 2026-03-20T11:46:23.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f9113b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f94d810:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e96f52900 20 trim: trimmed bucket instance dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:46:23.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e96f52900 15 stack 0x564c6f93f450 end 2026-03-20T11:46:23.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is done 2026-03-20T11:46:23.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f94d7e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e96f52900 4 starting trim on bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:23.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f963900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f963900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e96f52900 20 enqueued request req=0x564c6f81c970 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e96f52900 20 req: 0x564c6f81c970 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e47fe7640 20 dequeued request req=0x564c6f81c970 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e47fe7640 20 RGWWQ: empty 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e47fe7640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 : miss 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e47fe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is io blocked 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e47fe7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e47fe7640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 info.flags=0x17 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e47fe7640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 to cache LRU end 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e47fe7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e47fe7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e47fe7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e47fe7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-51 : miss 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e47fe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e47fe7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e47fe7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-51 info.flags=0x0 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e47fe7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-51 to cache LRU end 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e47fe7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-51 : miss 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.178+0000 7f4e47fe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e47fe7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e47fe7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-51 info.flags=0x0 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e47fe7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-51 to cache LRU end 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e47fe7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11): adding source pipe: {s={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e47fe7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11): adding dest pipe: {s={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e47fe7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e47fe7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e47fe7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f963900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f963900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f963900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f963900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f94d7e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f9113b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f94d7e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f90e430:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f90e430:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = e2ca8104acc01002a2de225420d48085f87bc4b511508e7b62361cdf2f85addf 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:e2ca8104acc01002a2de225420d48085f87bc4b511508e7b62361cdf2f85addf 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = a395cce549ff8444f6f1a22ccf56f558887880efd468cb9ba786ee9ccd37a1ed 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a395cce549ff8444f6f1a22ccf56f558887880efd468cb9ba786ee9ccd37a1ed 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=22, curl_handle=0x564c6f81eb70 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4df2ffd640 20 link_request req_data=0x564c6f81f660 req_data->id=22, curl_handle=0x564c6f81eb70 2026-03-20T11:46:23.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.179+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is io blocked 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f914100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f914100:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f4a5950:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e96f52900 15 stack 0x564c6f9106c0 end 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e96f52900 20 run: stack=0x564c6f9106c0 is done 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e96f52900 20 collect(): s=0x564c6f4a7b60 stack=0x564c6f9106c0 is complete 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e96f52900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e96f52900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e96f52900 15 stack 0x564c6f4a7b60 end 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e96f52900 20 stack->operate() returned ret=-22 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is done 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e96f52900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e96f52900 4 starting trim on bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f963900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f963900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e96f52900 20 enqueued request req=0x564c6f4a0b10 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e96f52900 20 req: 0x564c6f4a0b10 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is io blocked 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e527fc640 20 dequeued request req=0x564c6f4a0b10 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e527fc640 20 RGWWQ: empty 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e527fc640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 : miss 2026-03-20T11:46:23.179 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e527fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e527fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e527fc640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 info.flags=0x17 2026-03-20T11:46:23.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e527fc640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 to cache LRU end 2026-03-20T11:46:23.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e527fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e527fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e527fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:23.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e527fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-55 : miss 2026-03-20T11:46:23.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e527fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e527fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e527fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-55 info.flags=0x0 2026-03-20T11:46:23.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e527fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-55 to cache LRU end 2026-03-20T11:46:23.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e527fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-55 : miss 2026-03-20T11:46:23.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.182+0000 7f4e527fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e527fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e527fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-55 info.flags=0x0 2026-03-20T11:46:23.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e527fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-55 to cache LRU end 2026-03-20T11:46:23.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e527fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding source pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e527fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding dest pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e527fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e527fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e527fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:23.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f963900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f963900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f963900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f963900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f9563d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.180 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f952340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f952340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 7b2b05b456a7485bce1638f0ddf40ff59b3c153c2410eab0034b03f103f063e0 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:7b2b05b456a7485bce1638f0ddf40ff59b3c153c2410eab0034b03f103f063e0 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 82e897f7e4b125d3a33c9875754bad791e5d7cabad78e8e20423cc32721991fc 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=82e897f7e4b125d3a33c9875754bad791e5d7cabad78e8e20423cc32721991fc 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=23, curl_handle=0x564c6f915d30 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4df2ffd640 20 link_request req_data=0x564c6f954490 req_data->id=23, curl_handle=0x564c6f915d30 2026-03-20T11:46:23.181 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.183+0000 7f4e96f52900 20 run: stack=0x564c6f9106c0 is io blocked 2026-03-20T11:46:23.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f95eeb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f95eeb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:23.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f95b440:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e96f52900 15 stack 0x564c6f94b500 end 2026-03-20T11:46:23.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e96f52900 20 run: stack=0x564c6f94b500 is done 2026-03-20T11:46:23.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e96f52900 20 collect(): s=0x564c6f8132d0 stack=0x564c6f94b500 is complete 2026-03-20T11:46:23.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e96f52900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:46:23.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e96f52900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:23.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:23.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e96f52900 15 stack 0x564c6f8132d0 end 2026-03-20T11:46:23.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e96f52900 20 stack->operate() returned ret=-22 2026-03-20T11:46:23.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is done 2026-03-20T11:46:23.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e96f52900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:23.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e96f52900 4 starting trim on bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:23.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f961320:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f961320:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e96f52900 20 enqueued request req=0x564c6f90ec10 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e96f52900 20 req: 0x564c6f90ec10 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is io blocked 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e477e6640 20 dequeued request req=0x564c6f90ec10 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e477e6640 20 RGWWQ: empty 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e477e6640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 : miss 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.215+0000 7f4e477e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e477e6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e477e6640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 info.flags=0x17 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e477e6640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 to cache LRU end 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e477e6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e477e6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e477e6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e477e6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-28 : miss 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e477e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e477e6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e477e6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-28 info.flags=0x0 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e477e6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-28 to cache LRU end 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e477e6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-28 : miss 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e477e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e477e6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e477e6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-28 info.flags=0x0 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e477e6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-28 to cache LRU end 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e477e6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding source pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e477e6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding dest pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e477e6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e477e6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e477e6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f961320:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f961320:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f961320:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f961320:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f95edc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f961320:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f961320:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 73199aade528309b3ac9a566228341359587637d19902ba99b5fd66bf7e77805 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:73199aade528309b3ac9a566228341359587637d19902ba99b5fd66bf7e77805 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 0c74644c8a07547ed0f2003a3d0e65ed152e56185461fecd144ba213613eac87 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=0c74644c8a07547ed0f2003a3d0e65ed152e56185461fecd144ba213613eac87 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=24, curl_handle=0x564c6f823420 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4e96f52900 20 run: stack=0x564c6f94b500 is io blocked 2026-03-20T11:46:23.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.216+0000 7f4df2ffd640 20 link_request req_data=0x564c6f918e60 req_data->id=24, curl_handle=0x564c6f823420 2026-03-20T11:46:23.215 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:23.216+0000 7fcfc59d9640 -1 req 8556964848092715228 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f91b520:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e96f52900 15 stack 0x564c6f907e20 end 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is done 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e96f52900 20 collect(): s=0x564c6f90d800 stack=0x564c6f907e20 is complete 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e96f52900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e96f52900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e96f52900 15 stack 0x564c6f90d800 end 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e96f52900 20 stack->operate() returned ret=-22 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e96f52900 20 run: stack=0x564c6f90d800 is done 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e96f52900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e96f52900 4 starting trim on bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f963900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f963900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e96f52900 20 enqueued request req=0x564c6f4a34a0 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e96f52900 20 req: 0x564c6f4a34a0 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e96f52900 20 run: stack=0x564c6f90d800 is io blocked 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e46fe5640 20 dequeued request req=0x564c6f4a34a0 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e46fe5640 20 RGWWQ: empty 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e46fe5640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 : miss 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e46fe5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e46fe5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e46fe5640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 info.flags=0x17 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e46fe5640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 to cache LRU end 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e46fe5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e46fe5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e46fe5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e46fe5640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-31 : miss 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e46fe5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e46fe5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e46fe5640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-31 info.flags=0x0 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e46fe5640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-31 to cache LRU end 2026-03-20T11:46:23.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e46fe5640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-31 : miss 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.218+0000 7f4e46fe5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e46fe5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e46fe5640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-31 info.flags=0x0 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e46fe5640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-31 to cache LRU end 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e46fe5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding source pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e46fe5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding dest pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e46fe5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e46fe5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e46fe5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f963900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f963900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f963900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f963900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f91b520:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 20 run: stack=0x564c6f90d800 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.216 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.217 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.217 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.217 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.217 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.217 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.217 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 591f37583add487ac07e3dba8af60fc9b7ebc0e013199b67a82891e47e636f30 2026-03-20T11:46:23.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.217 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.217 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.217 INFO:teuthology.orchestra.run.vm02.stderr:591f37583add487ac07e3dba8af60fc9b7ebc0e013199b67a82891e47e636f30 2026-03-20T11:46:23.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 4b408ecd4ecbfec23091f2a2a05b28ff8ebb93dcdc472f9d0865e70f0d0243b2 2026-03-20T11:46:23.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=4b408ecd4ecbfec23091f2a2a05b28ff8ebb93dcdc472f9d0865e70f0d0243b2 2026-03-20T11:46:23.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=25, curl_handle=0x564c6f911bf0 2026-03-20T11:46:23.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is io blocked 2026-03-20T11:46:23.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.219+0000 7f4df2ffd640 20 link_request req_data=0x564c6f912300 req_data->id=25, curl_handle=0x564c6f911bf0 2026-03-20T11:46:23.217 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:23.219+0000 7fd06f32c640 -1 req 211308987224590611 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:23.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f90e430:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f90e430:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f90e430:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f90e430:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f9113b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 15 stack 0x564c6f94bd80 end 2026-03-20T11:46:23.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is done 2026-03-20T11:46:23.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f94d7e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 collect(): s=0x564c6f93f450 stack=0x564c6f94bd80 is complete 2026-03-20T11:46:23.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 10 trimming bilogs for bucket=:dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) markers=[00000000004.41.6,00000000002.16.1,00000000004.28.6,00000000002.34.1,00000000004.45.6,00000000002.40.1,00000000002.39.1,00000000004.52.6,00000000002.73.1,00000000002.41.1,00000000002.74.1], shards=11 2026-03-20T11:46:23.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f957ad0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 10 trimming bilog shard 0 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000004.41.6 2026-03-20T11:46:23.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 10 trimming bilog shard 1 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.16.1 2026-03-20T11:46:23.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 10 trimming bilog shard 2 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000004.28.6 2026-03-20T11:46:23.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 10 trimming bilog shard 3 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.34.1 2026-03-20T11:46:23.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 10 trimming bilog shard 4 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000004.45.6 2026-03-20T11:46:23.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 10 trimming bilog shard 5 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.40.1 2026-03-20T11:46:23.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 10 trimming bilog shard 6 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.39.1 2026-03-20T11:46:23.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 10 trimming bilog shard 7 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000004.52.6 2026-03-20T11:46:23.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 10 trimming bilog shard 8 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.73.1 2026-03-20T11:46:23.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 10 trimming bilog shard 9 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.41.1 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 10 trimming bilog shard 10 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.74.1 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f4a5950:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f4a02e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f94b640:op=0x564c6f919d50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f960a30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f909ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f4a4760:op=0x564c6f821840:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f90a760:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f9643d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f94f030:op=0x564c6f966d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f968c90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f91c260:op=0x564c6f969490:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f4a5950:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.0 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is io blocked 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f4a02e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.1 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 run: stack=0x564c6f8210c0 is io blocked 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f94b640:op=0x564c6f919d50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.2 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 run: stack=0x564c6f94b640 is io blocked 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f960a30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.3 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 run: stack=0x564c6f953ef0 is io blocked 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f909ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.4 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is io blocked 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f4a4760:op=0x564c6f821840:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.5 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 run: stack=0x564c6f4a4760 is io blocked 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f90a760:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.6 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is io blocked 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f9643d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.7 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 run: stack=0x564c6f91c0c0 is io blocked 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f94f030:op=0x564c6f966d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.8 2026-03-20T11:46:23.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 run: stack=0x564c6f94f030 is io blocked 2026-03-20T11:46:23.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f968c90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.9 2026-03-20T11:46:23.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 run: stack=0x564c6f94c000 is io blocked 2026-03-20T11:46:23.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 cr:s=0x564c6f91c260:op=0x564c6f969490:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.10 2026-03-20T11:46:23.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.221+0000 7f4e96f52900 20 run: stack=0x564c6f91c260 is io blocked 2026-03-20T11:46:23.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f94b640:op=0x564c6f919d50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f94b640:op=0x564c6f919d50:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 15 stack 0x564c6f94b640 end 2026-03-20T11:46:23.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 run: stack=0x564c6f94b640 is done 2026-03-20T11:46:23.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f957ad0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f4a5950:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f4a5950:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 15 stack 0x564c6f94bd80 end 2026-03-20T11:46:23.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is done 2026-03-20T11:46:23.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f957ad0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f9643d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f9643d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 15 stack 0x564c6f91c0c0 end 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 run: stack=0x564c6f91c0c0 is done 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f957ad0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f90a760:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f90a760:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 15 stack 0x564c6f4a77d0 end 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is done 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f960a30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f960a30:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 15 stack 0x564c6f953ef0 end 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 run: stack=0x564c6f953ef0 is done 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f4a02e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f4a02e0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 15 stack 0x564c6f8210c0 end 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 run: stack=0x564c6f8210c0 is done 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f957ad0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f909ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f909ae0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 15 stack 0x564c6f821700 end 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is done 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f957ad0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f957ad0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f957ad0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f4a4760:op=0x564c6f821840:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f4a4760:op=0x564c6f821840:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 15 stack 0x564c6f4a4760 end 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 run: stack=0x564c6f4a4760 is done 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f957ad0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f968c90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f968c90:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 15 stack 0x564c6f94c000 end 2026-03-20T11:46:23.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.222 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 run: stack=0x564c6f94c000 is done 2026-03-20T11:46:23.222 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f957ad0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.222 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.222 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f94f030:op=0x564c6f966d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.222 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f94f030:op=0x564c6f966d20:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.222 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 15 stack 0x564c6f94f030 end 2026-03-20T11:46:23.222 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.222 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 run: stack=0x564c6f94f030 is done 2026-03-20T11:46:23.222 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f957ad0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.222 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.223+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e96f52900 20 cr:s=0x564c6f91c260:op=0x564c6f969490:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e96f52900 20 cr:s=0x564c6f91c260:op=0x564c6f969490:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e96f52900 15 stack 0x564c6f91c260 end 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e96f52900 20 run: stack=0x564c6f91c260 is done 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f957ad0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e96f52900 20 cr:s=0x564c6f93f450:op=0x564c6f94d7e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e96f52900 20 trim: trimmed bucket instance dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e96f52900 15 stack 0x564c6f93f450 end 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e96f52900 20 run: stack=0x564c6f93f450 is done 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f94d7e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e96f52900 4 starting trim on bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f95d900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f95d900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e96f52900 20 enqueued request req=0x564c6f90b5b0 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e96f52900 20 req: 0x564c6f90b5b0 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is io blocked 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e467e4640 20 dequeued request req=0x564c6f90b5b0 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e467e4640 20 RGWWQ: empty 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e467e4640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 : miss 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e467e4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e467e4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e467e4640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 info.flags=0x17 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e467e4640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 to cache LRU end 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e467e4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e467e4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e467e4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e467e4640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-39 : miss 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.224+0000 7f4e467e4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f952340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f952340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.223 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f952340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f952340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f9563d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 15 stack 0x564c6f9106c0 end 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 20 run: stack=0x564c6f9106c0 is done 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 20 collect(): s=0x564c6f4a7b60 stack=0x564c6f9106c0 is complete 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 10 trimming bilogs for bucket=:dipepu-55[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15]) markers=[,,,,,00000000006.98.6,,,,,], shards=11 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f965000:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 10 trimming bilog shard 5 of :dipepu-55[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15]) at marker 00000000006.98.6 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f9643d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f9643d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15.5 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 20 run: stack=0x564c6f9106c0 is io blocked 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e467e4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e467e4640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-39 info.flags=0x0 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e467e4640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-39 to cache LRU end 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e467e4640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-39 : miss 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e467e4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e467e4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e467e4640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-39 info.flags=0x0 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e467e4640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-39 to cache LRU end 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e467e4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding source pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e467e4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding dest pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e467e4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e467e4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e467e4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f95d900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f95d900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f95d900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f95d900:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f94d7e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f9113b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f94d7e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f90e430:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f90e430:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.225+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.224 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.225 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.225 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.225 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.225 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.225 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.225 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.225 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.225 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.225 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 6ef5d3a6b63818308c2bb1f20701ddf05328dab1908d12a10754212c0d6c0e72 2026-03-20T11:46:23.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.225 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.225 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.225 INFO:teuthology.orchestra.run.vm02.stderr:6ef5d3a6b63818308c2bb1f20701ddf05328dab1908d12a10754212c0d6c0e72 2026-03-20T11:46:23.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 837f9c077f5e428edb427361c22edab64911ca2ccdc7b3baed218f70509835de 2026-03-20T11:46:23.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=837f9c077f5e428edb427361c22edab64911ca2ccdc7b3baed218f70509835de 2026-03-20T11:46:23.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=26, curl_handle=0x564c6f915d30 2026-03-20T11:46:23.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is io blocked 2026-03-20T11:46:23.225 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4df2ffd640 20 link_request req_data=0x564c6f954490 req_data->id=26, curl_handle=0x564c6f915d30 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f9643d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f9643d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 15 stack 0x564c6f9106c0 end 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 20 run: stack=0x564c6f9106c0 is done 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f965000:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 20 trim: trimmed bucket instance dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 15 stack 0x564c6f4a7b60 end 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is done 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 4 starting trim on bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f95ca30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f95ca30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 20 enqueued request req=0x564c6f4a0b10 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 20 req: 0x564c6f4a0b10 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is io blocked 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e45fe3640 20 dequeued request req=0x564c6f4a0b10 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e45fe3640 20 RGWWQ: empty 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e45fe3640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 : miss 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.226+0000 7f4e45fe3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.227+0000 7f4e45fe3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.227+0000 7f4e45fe3640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 info.flags=0x17 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.227+0000 7f4e45fe3640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 to cache LRU end 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.227+0000 7f4e45fe3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.227+0000 7f4e45fe3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.227+0000 7f4e45fe3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.227+0000 7f4e45fe3640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-8 : miss 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.227+0000 7f4e45fe3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e45fe3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e45fe3640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-8 info.flags=0x0 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e45fe3640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-8 to cache LRU end 2026-03-20T11:46:23.226 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e45fe3640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-8 : miss 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e45fe3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e45fe3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e45fe3640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-8 info.flags=0x0 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e45fe3640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-8 to cache LRU end 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e45fe3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding source pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e45fe3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding dest pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e45fe3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e45fe3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e45fe3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f95ca30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f95ca30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f95ca30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f95ca30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f9657e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f96a070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f96a070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 348928449eccd68e2a77113648f1ce8c8962aab3983d47e1492454fd327dcc28 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:348928449eccd68e2a77113648f1ce8c8962aab3983d47e1492454fd327dcc28 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 92e245bd7b8eb93edad8372f4a9ef245134e0a7718eeabf595619da8bb0d6dbb 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=92e245bd7b8eb93edad8372f4a9ef245134e0a7718eeabf595619da8bb0d6dbb 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=27, curl_handle=0x564c6f81eb70 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4e96f52900 20 run: stack=0x564c6f9106c0 is io blocked 2026-03-20T11:46:23.227 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.228+0000 7f4df2ffd640 20 link_request req_data=0x564c6f81f660 req_data->id=27, curl_handle=0x564c6f81eb70 2026-03-20T11:46:23.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f961320:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f961320:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:23.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f95edc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e96f52900 15 stack 0x564c6f94b500 end 2026-03-20T11:46:23.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e96f52900 20 run: stack=0x564c6f94b500 is done 2026-03-20T11:46:23.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e96f52900 20 collect(): s=0x564c6f8132d0 stack=0x564c6f94b500 is complete 2026-03-20T11:46:23.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e96f52900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:23.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e96f52900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:23.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:23.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e96f52900 15 stack 0x564c6f8132d0 end 2026-03-20T11:46:23.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e96f52900 20 stack->operate() returned ret=-22 2026-03-20T11:46:23.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is done 2026-03-20T11:46:23.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e96f52900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:23.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e96f52900 4 starting trim on bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:23.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f961320:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f961320:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e96f52900 20 enqueued request req=0x564c6f90ec10 2026-03-20T11:46:23.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e96f52900 20 req: 0x564c6f90ec10 2026-03-20T11:46:23.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is io blocked 2026-03-20T11:46:23.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e457e2640 20 dequeued request req=0x564c6f90ec10 2026-03-20T11:46:23.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e457e2640 20 RGWWQ: empty 2026-03-20T11:46:23.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e457e2640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 : miss 2026-03-20T11:46:23.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.258+0000 7f4e457e2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.259+0000 7f4e457e2640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.259+0000 7f4e457e2640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 info.flags=0x17 2026-03-20T11:46:23.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.259+0000 7f4e457e2640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 to cache LRU end 2026-03-20T11:46:23.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.259+0000 7f4e457e2640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.259+0000 7f4e457e2640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.259+0000 7f4e457e2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:23.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.259+0000 7f4e457e2640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-16 : miss 2026-03-20T11:46:23.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.259+0000 7f4e457e2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.259+0000 7f4e457e2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.259+0000 7f4e457e2640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-16 info.flags=0x0 2026-03-20T11:46:23.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.259+0000 7f4e457e2640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-16 to cache LRU end 2026-03-20T11:46:23.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.259+0000 7f4e457e2640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-16 : miss 2026-03-20T11:46:23.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.259+0000 7f4e457e2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e457e2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e457e2640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-16 info.flags=0x0 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e457e2640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-16 to cache LRU end 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e457e2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding source pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e457e2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding dest pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f91b520:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 15 stack 0x564c6f907e20 end 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is done 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 collect(): s=0x564c6f90d800 stack=0x564c6f907e20 is complete 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 cr:s=0x564c6f90d800:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 15 stack 0x564c6f90d800 end 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 stack->operate() returned ret=-22 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 run: stack=0x564c6f90d800 is done 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e457e2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e457e2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 4 starting trim on bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f95ca10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f95ca10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 enqueued request req=0x564c6f4a34a0 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 req: 0x564c6f4a34a0 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is io blocked 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e44fe1640 20 dequeued request req=0x564c6f4a34a0 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e44fe1640 20 RGWWQ: empty 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e44fe1640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 : miss 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e44fe1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e457e2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f961320:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f961320:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f961320:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f961320:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f95edc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 5ad03dcbfb6660393da0cfc3fff424aa56f7a4e1c288cd1a39af88ae41e806c5 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e44fe1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=314 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:5ad03dcbfb6660393da0cfc3fff424aa56f7a4e1c288cd1a39af88ae41e806c5 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e44fe1640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 info.flags=0x17 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e44fe1640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 to cache LRU end 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e44fe1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e44fe1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e44fe1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = e8eb27aa4bcdcfbb86fdf5a40d298546651c9c21b147989d2524d7c89506aff7 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=e8eb27aa4bcdcfbb86fdf5a40d298546651c9c21b147989d2524d7c89506aff7 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e44fe1640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-43 : miss 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e44fe1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=28, curl_handle=0x564c6f911bf0 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4e96f52900 20 run: stack=0x564c6f94b500 is io blocked 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.260+0000 7f4df2ffd640 20 link_request req_data=0x564c6f912300 req_data->id=28, curl_handle=0x564c6f911bf0 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e44fe1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e44fe1640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-43 info.flags=0x0 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e44fe1640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-43 to cache LRU end 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e44fe1640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-43 : miss 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e44fe1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e44fe1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e44fe1640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-43 info.flags=0x0 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e44fe1640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-43 to cache LRU end 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e44fe1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding source pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e44fe1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding dest pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e44fe1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e44fe1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e44fe1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:23.259 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:23.261+0000 7fd045ad9640 -1 req 6840689667191508530 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f95ca10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f95ca10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f95ca10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f95ca10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f95a530:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f952340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f952340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.259 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = d98908d0c260bacbb889b488947764b144b33a21aaa805b192e9f18dfd665a58 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:d98908d0c260bacbb889b488947764b144b33a21aaa805b192e9f18dfd665a58 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 04f4fe57e5abeecb8092fe64ce190710b96ed264939f742eb098dd7fad770c6a 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=04f4fe57e5abeecb8092fe64ce190710b96ed264939f742eb098dd7fad770c6a 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=29, curl_handle=0x564c6f823420 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4e96f52900 20 run: stack=0x564c6f953ef0 is io blocked 2026-03-20T11:46:23.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.261+0000 7f4df2ffd640 20 link_request req_data=0x564c6f918e60 req_data->id=29, curl_handle=0x564c6f823420 2026-03-20T11:46:23.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.269+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f90e430:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.269+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f90e430:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.269+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f90e430:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.269+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f90e430:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.269+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f9113b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.269+0000 7f4e96f52900 15 stack 0x564c6f821700 end 2026-03-20T11:46:23.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.269+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is done 2026-03-20T11:46:23.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.269+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f94d7e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.269+0000 7f4e96f52900 20 collect(): s=0x564c6f4a77d0 stack=0x564c6f821700 is complete 2026-03-20T11:46:23.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.269+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.269+0000 7f4e96f52900 10 trimming bilogs for bucket=:dipepu-39[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44]) markers=[,,,,,,,,00000000001.10.6,,], shards=11 2026-03-20T11:46:23.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.269+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f9577f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.269+0000 7f4e96f52900 10 trimming bilog shard 8 of :dipepu-39[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44]) at marker 00000000001.10.6 2026-03-20T11:46:23.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.269+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.269+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f90b800:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.269+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f90b800:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.269+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44.8 2026-03-20T11:46:23.266 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.269+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is io blocked 2026-03-20T11:46:23.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.270+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f90b800:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.270+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f90b800:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.270+0000 7f4e96f52900 15 stack 0x564c6f821700 end 2026-03-20T11:46:23.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.270+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.270+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is done 2026-03-20T11:46:23.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.270+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f9577f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.270+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f94d7e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.270+0000 7f4e96f52900 20 trim: trimmed bucket instance dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:23.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.270+0000 7f4e96f52900 15 stack 0x564c6f4a77d0 end 2026-03-20T11:46:23.267 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.270+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is done 2026-03-20T11:46:23.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.270+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.270+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.270+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f95b6a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.270+0000 7f4e96f52900 4 starting trim on bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:23.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.270+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f9643d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.270+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f9643d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.270+0000 7f4e96f52900 20 enqueued request req=0x564c6f90b5b0 2026-03-20T11:46:23.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.270+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.270+0000 7f4e96f52900 20 req: 0x564c6f90b5b0 2026-03-20T11:46:23.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.270+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is io blocked 2026-03-20T11:46:23.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.270+0000 7f4e447e0640 20 dequeued request req=0x564c6f90b5b0 2026-03-20T11:46:23.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.270+0000 7f4e447e0640 20 RGWWQ: empty 2026-03-20T11:46:23.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.270+0000 7f4e447e0640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 : miss 2026-03-20T11:46:23.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.270+0000 7f4e447e0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f96a070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e447e0640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e447e0640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 info.flags=0x17 2026-03-20T11:46:23.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e447e0640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 to cache LRU end 2026-03-20T11:46:23.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e447e0640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e447e0640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f96a070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f96a070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f96a070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f9657e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e447e0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:23.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 15 stack 0x564c6f9106c0 end 2026-03-20T11:46:23.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 run: stack=0x564c6f9106c0 is done 2026-03-20T11:46:23.268 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 collect(): s=0x564c6f4a7b60 stack=0x564c6f9106c0 is complete 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 10 trimming bilogs for bucket=:dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) markers=[,00000000002.12.6,,00000000001.5.6,,00000000001.4.6,00000000002.12.6,00000000001.8.6,00000000001.14.6,00000000001.9.6,], shards=11 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e447e0640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-10 : miss 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f9113b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 10 trimming bilog shard 1 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000002.12.6 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e447e0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 10 trimming bilog shard 3 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.5.6 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 10 trimming bilog shard 5 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.4.6 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 10 trimming bilog shard 6 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000002.12.6 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 10 trimming bilog shard 7 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.8.6 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 10 trimming bilog shard 8 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.14.6 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 10 trimming bilog shard 9 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.9.6 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f821840:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f919d50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f969e70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f9657e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f90a760:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 cr:s=0x564c6f94b640:op=0x564c6f969080:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f81ccf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f821840:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.1 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e447e0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e447e0640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-10 info.flags=0x0 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e447e0640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-10 to cache LRU end 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e447e0640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-10 : miss 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e447e0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 run: stack=0x564c6f9106c0 is io blocked 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f919d50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.3 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is io blocked 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f969e70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.5 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 run: stack=0x564c6f8210c0 is io blocked 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f9657e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.6 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 run: stack=0x564c6f91c0c0 is io blocked 2026-03-20T11:46:23.269 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f90a760:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.7 2026-03-20T11:46:23.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is io blocked 2026-03-20T11:46:23.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 cr:s=0x564c6f94b640:op=0x564c6f969080:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.8 2026-03-20T11:46:23.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 run: stack=0x564c6f94b640 is io blocked 2026-03-20T11:46:23.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f81ccf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.9 2026-03-20T11:46:23.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e96f52900 20 run: stack=0x564c6f4a1590 is io blocked 2026-03-20T11:46:23.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e447e0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e447e0640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-10 info.flags=0x0 2026-03-20T11:46:23.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e447e0640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-10 to cache LRU end 2026-03-20T11:46:23.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e447e0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding source pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e447e0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding dest pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e447e0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e447e0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.271+0000 7f4e447e0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:23.270 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f9643d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f9643d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f9643d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f9643d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f95b6a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 20 cr:s=0x564c6f960140:op=0x564c6f81d4f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f95b6a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 20 cr:s=0x564c6f960140:op=0x564c6f965000:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 20 cr:s=0x564c6f960140:op=0x564c6f965000:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 08aa43ca814dd44bdfa822367668848554897af00f7183d55e7d0b39a60295f5 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:08aa43ca814dd44bdfa822367668848554897af00f7183d55e7d0b39a60295f5 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 7b5a6346467cf3e3c3872a75ad29feb734050612c8a546fb918e70037f036c1a 2026-03-20T11:46:23.271 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=7b5a6346467cf3e3c3872a75ad29feb734050612c8a546fb918e70037f036c1a 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=30, curl_handle=0x564c6f81eb70 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4e96f52900 20 run: stack=0x564c6f960140 is io blocked 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.272+0000 7f4df2ffd640 20 link_request req_data=0x564c6f81f660 req_data->id=30, curl_handle=0x564c6f81eb70 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.273+0000 7f4e96f52900 20 cr:s=0x564c6f94b640:op=0x564c6f969080:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.273+0000 7f4e96f52900 20 cr:s=0x564c6f94b640:op=0x564c6f969080:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.273+0000 7f4e96f52900 15 stack 0x564c6f94b640 end 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.273+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.273+0000 7f4e96f52900 20 run: stack=0x564c6f94b640 is done 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.273+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f9113b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.273+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.273+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f969e70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.273+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f969e70:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.273+0000 7f4e96f52900 15 stack 0x564c6f8210c0 end 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.273+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.273+0000 7f4e96f52900 20 run: stack=0x564c6f8210c0 is done 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.273+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f9113b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.273+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.273+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f821840:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.273+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f821840:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.273+0000 7f4e96f52900 15 stack 0x564c6f9106c0 end 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 run: stack=0x564c6f9106c0 is done 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f9113b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f90a760:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f90a760:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 15 stack 0x564c6f94bd80 end 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is done 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f9113b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.272 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.272 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:23.274+0000 7fd01026e640 -1 req 9208613595543314138 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f919d50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f919d50:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 15 stack 0x564c6f821700 end 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is done 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f9113b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f81ccf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f81ccf0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 15 stack 0x564c6f4a1590 end 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 run: stack=0x564c6f4a1590 is done 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f9657e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f9657e0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 15 stack 0x564c6f91c0c0 end 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 run: stack=0x564c6f91c0c0 is done 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f9113b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f9113b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 trim: trimmed bucket instance dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 15 stack 0x564c6f4a7b60 end 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is done 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 4 starting trim on bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f957b40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f957b40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 enqueued request req=0x564c6f908e00 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 req: 0x564c6f908e00 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is io blocked 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e53fff640 20 dequeued request req=0x564c6f908e00 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e53fff640 20 RGWWQ: empty 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e53fff640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 : miss 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e53fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e53fff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e53fff640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 info.flags=0x17 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e53fff640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 to cache LRU end 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e53fff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e53fff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e53fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e53fff640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-3 : miss 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e53fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e53fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e53fff640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-3 info.flags=0x0 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e53fff640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-3 to cache LRU end 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e53fff640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-3 : miss 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.274+0000 7f4e53fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e53fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e53fff640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-3 info.flags=0x0 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e53fff640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-3 to cache LRU end 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e53fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding source pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e53fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding dest pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e53fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e53fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e53fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f957b40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f957b40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f957b40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f957b40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f9113b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f960a30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f960a30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = e470d6ac1fe43226137766128f5587d246e1e2baeb18d2ab984019e336459633 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:e470d6ac1fe43226137766128f5587d246e1e2baeb18d2ab984019e336459633 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 83556c10effb7bfb3217287034d432c5753a5d781f09624e98049b2894ebdd25 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=83556c10effb7bfb3217287034d432c5753a5d781f09624e98049b2894ebdd25 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=31, curl_handle=0x564c6f915d30 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4e96f52900 20 run: stack=0x564c6f91c0c0 is io blocked 2026-03-20T11:46:23.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.275+0000 7f4df2ffd640 20 link_request req_data=0x564c6f954490 req_data->id=31, curl_handle=0x564c6f915d30 2026-03-20T11:46:23.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f952340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f952340:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:23.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f95a530:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 15 stack 0x564c6f953ef0 end 2026-03-20T11:46:23.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 run: stack=0x564c6f953ef0 is done 2026-03-20T11:46:23.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 collect(): s=0x564c6f907e20 stack=0x564c6f953ef0 is complete 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 15 stack 0x564c6f907e20 end 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 stack->operate() returned ret=-22 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is done 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f95edc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 15 stack 0x564c6f94b500 end 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 run: stack=0x564c6f94b500 is done 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 collect(): s=0x564c6f8132d0 stack=0x564c6f94b500 is complete 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 10 trimming bilogs for bucket=:dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) markers=[,,,,,00000000003.17.10,,,00000000031.93.9,,], shards=11 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 4 starting trim on bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f95edc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 10 trimming bilog shard 5 of :dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) at marker 00000000003.17.10 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 10 trimming bilog shard 8 of :dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) at marker 00000000031.93.9 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f963740:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f94d7e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f821840:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f963740:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 enqueued request req=0x564c6f4a1710 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 req: 0x564c6f4a1710 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is io blocked 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f94d7e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21.5 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e51ffb640 20 dequeued request req=0x564c6f4a1710 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e51ffb640 20 RGWWQ: empty 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 run: stack=0x564c6f94b500 is io blocked 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f821840:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21.8 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e51ffb640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 : miss 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e96f52900 20 run: stack=0x564c6f953ef0 is io blocked 2026-03-20T11:46:23.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.303+0000 7f4e51ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.304+0000 7f4e51ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:46:23.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.304+0000 7f4e51ffb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 info.flags=0x17 2026-03-20T11:46:23.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.304+0000 7f4e51ffb640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 to cache LRU end 2026-03-20T11:46:23.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.304+0000 7f4e51ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.304+0000 7f4e51ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.304+0000 7f4e51ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:23.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.304+0000 7f4e51ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-9 : miss 2026-03-20T11:46:23.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.304+0000 7f4e51ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.304+0000 7f4e51ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.304+0000 7f4e51ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-9 info.flags=0x0 2026-03-20T11:46:23.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.304+0000 7f4e51ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-9 to cache LRU end 2026-03-20T11:46:23.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.304+0000 7f4e51ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-9 : miss 2026-03-20T11:46:23.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.304+0000 7f4e51ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e51ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e51ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-9 info.flags=0x0 2026-03-20T11:46:23.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e51ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-9 to cache LRU end 2026-03-20T11:46:23.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e51ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding source pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e51ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding dest pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e51ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e51ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e51ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:23.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f963740:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f963740:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f821840:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f821840:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 15 stack 0x564c6f953ef0 end 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 run: stack=0x564c6f953ef0 is done 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f94d7e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f94d7e0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 15 stack 0x564c6f94b500 end 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 run: stack=0x564c6f94b500 is done 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f963740:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f95edc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f963740:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f95edc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 trim: trimmed bucket instance dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 15 stack 0x564c6f8132d0 end 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is done 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f95edc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f966520:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 4 starting trim on bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f966520:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 689e4f880f5984ecd8417135601ec07c707b236a17b176714f38deeae4edeb38 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:689e4f880f5984ecd8417135601ec07c707b236a17b176714f38deeae4edeb38 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = f183d71f297643f83334a75a2bbb92b5b5862a25367a6d798d401d41312efc66 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=f183d71f297643f83334a75a2bbb92b5b5862a25367a6d798d401d41312efc66 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.305 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=32, curl_handle=0x564c6f823420 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4df2ffd640 20 link_request req_data=0x564c6f918e60 req_data->id=32, curl_handle=0x564c6f823420 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 run: stack=0x564c6f953ef0 is io blocked 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f963740:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f963740:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 enqueued request req=0x564c6f4a0b10 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 req: 0x564c6f4a0b10 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is io blocked 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e537fe640 20 dequeued request req=0x564c6f4a0b10 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e537fe640 20 RGWWQ: empty 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e537fe640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 : miss 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e537fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e537fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e537fe640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 info.flags=0x17 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e537fe640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 to cache LRU end 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e537fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e537fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e537fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e537fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-38 : miss 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.305+0000 7f4e537fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e537fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e537fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-38 info.flags=0x0 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e537fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-38 to cache LRU end 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e537fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-38 : miss 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e537fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e537fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e537fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-38 info.flags=0x0 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e537fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-38 to cache LRU end 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e537fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding source pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e537fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding dest pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e537fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e537fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e537fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f963740:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f963740:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f963740:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f963740:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f947990:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f95a530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f95a530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.306 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.307 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 910141ce913057682c958bf551e0bce75458f7759b046a3e8aebd5294b2f4e7b 2026-03-20T11:46:23.307 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.307 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.307 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.307 INFO:teuthology.orchestra.run.vm02.stderr:910141ce913057682c958bf551e0bce75458f7759b046a3e8aebd5294b2f4e7b 2026-03-20T11:46:23.307 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.307 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.307 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.307 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.307 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = dbe46b14b7093c43338734f7f4a42ad54219b9af357d23624a0e0357bd995769 2026-03-20T11:46:23.307 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=dbe46b14b7093c43338734f7f4a42ad54219b9af357d23624a0e0357bd995769 2026-03-20T11:46:23.307 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.307 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.307 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.307 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=33, curl_handle=0x564c6f911bf0 2026-03-20T11:46:23.307 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4df2ffd640 20 link_request req_data=0x564c6f912300 req_data->id=33, curl_handle=0x564c6f911bf0 2026-03-20T11:46:23.307 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.306+0000 7f4e96f52900 20 run: stack=0x564c6f94b500 is io blocked 2026-03-20T11:46:23.311 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.315+0000 7f4e96f52900 20 cr:s=0x564c6f960140:op=0x564c6f965000:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.315+0000 7f4e96f52900 20 cr:s=0x564c6f960140:op=0x564c6f965000:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.315+0000 7f4e96f52900 20 cr:s=0x564c6f960140:op=0x564c6f965000:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.315+0000 7f4e96f52900 20 cr:s=0x564c6f960140:op=0x564c6f965000:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.315+0000 7f4e96f52900 20 cr:s=0x564c6f960140:op=0x564c6f81d4f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.315+0000 7f4e96f52900 15 stack 0x564c6f960140 end 2026-03-20T11:46:23.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.315+0000 7f4e96f52900 20 run: stack=0x564c6f960140 is done 2026-03-20T11:46:23.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.315+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f95b6a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.315+0000 7f4e96f52900 20 collect(): s=0x564c6f4a77d0 stack=0x564c6f960140 is complete 2026-03-20T11:46:23.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.315+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.315+0000 7f4e96f52900 10 trimming bilogs for bucket=:dipepu-10[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15]) markers=[,00000000002.9.6,,,,,,,,,], shards=11 2026-03-20T11:46:23.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.315+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f965000:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.315+0000 7f4e96f52900 10 trimming bilog shard 1 of :dipepu-10[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15]) at marker 00000000002.9.6 2026-03-20T11:46:23.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.315+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.315+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f95d630:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.315+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f95d630:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.315+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15.1 2026-03-20T11:46:23.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.315+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is io blocked 2026-03-20T11:46:23.312 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f960a30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f960a30:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f9113b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 15 stack 0x564c6f91c0c0 end 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 run: stack=0x564c6f91c0c0 is done 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 collect(): s=0x564c6f4a7b60 stack=0x564c6f91c0c0 is complete 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 15 stack 0x564c6f4a7b60 end 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 stack->operate() returned ret=-22 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is done 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 4 starting trim on bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f90a760:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f90a760:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 enqueued request req=0x564c6f908e00 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 req: 0x564c6f908e00 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is io blocked 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e517fa640 20 dequeued request req=0x564c6f908e00 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e517fa640 20 RGWWQ: empty 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e517fa640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 : miss 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e517fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e517fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e517fa640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 info.flags=0x17 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e517fa640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 to cache LRU end 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e517fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e517fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e517fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e517fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-15 : miss 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e517fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.313 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f95d630:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f95d630:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 15 stack 0x564c6f821700 end 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is done 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f965000:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f95b6a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 trim: trimmed bucket instance dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 15 stack 0x564c6f4a77d0 end 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is done 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f95b4c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 4 starting trim on bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 enqueued request req=0x564c6f90b5b0 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 req: 0x564c6f90b5b0 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is io blocked 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e52ffd640 20 dequeued request req=0x564c6f90b5b0 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e52ffd640 20 RGWWQ: empty 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e52ffd640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 : miss 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.316+0000 7f4e52ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e517fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e517fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-15 info.flags=0x0 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e517fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-15 to cache LRU end 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e517fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-15 : miss 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e517fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e52ffd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e52ffd640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 info.flags=0x17 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e52ffd640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 to cache LRU end 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e52ffd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e52ffd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e52ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e52ffd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-13 : miss 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e52ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e52ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e52ffd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-13 info.flags=0x0 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e52ffd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-13 to cache LRU end 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e52ffd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-13 : miss 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e52ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e517fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e517fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-15 info.flags=0x0 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e517fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-15 to cache LRU end 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e517fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding source pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e517fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding dest pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e517fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e517fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e517fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f90a760:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f90a760:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f90a760:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f90a760:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f9113b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f965000:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f965000:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = a71e05d5a2775692b91bdb349586067edb422001244d340b77e6fcb8318d13df 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:a71e05d5a2775692b91bdb349586067edb422001244d340b77e6fcb8318d13df 2026-03-20T11:46:23.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 44c3417663d14b6309871f7259a45b4e725c77a82ef3dcc5ce51dafb7c214202 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=44c3417663d14b6309871f7259a45b4e725c77a82ef3dcc5ce51dafb7c214202 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=34, curl_handle=0x564c6f915d30 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is io blocked 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.317+0000 7f4df2ffd640 20 link_request req_data=0x564c6f954490 req_data->id=34, curl_handle=0x564c6f915d30 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e52ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e52ffd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-13 info.flags=0x0 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e52ffd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-13 to cache LRU end 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e52ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18): adding source pipe: {s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e52ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18): adding dest pipe: {s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e52ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e52ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e52ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f95b4c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f90ba50:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f95b4c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f94d7e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f94d7e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.316 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.317 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.317 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.317 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.317 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.317 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 8dfe9c0b803b23c393658db5f606404fc9733b2e1fd019b05dd7b8bc386a2b74 2026-03-20T11:46:23.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.317 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.317 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.317 INFO:teuthology.orchestra.run.vm02.stderr:8dfe9c0b803b23c393658db5f606404fc9733b2e1fd019b05dd7b8bc386a2b74 2026-03-20T11:46:23.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 0bae6c45c3d96f637426f763451761d33849c73439ad3c66db671a3646f4d844 2026-03-20T11:46:23.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=0bae6c45c3d96f637426f763451761d33849c73439ad3c66db671a3646f4d844 2026-03-20T11:46:23.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=35, curl_handle=0x564c6f81eb70 2026-03-20T11:46:23.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4e96f52900 20 run: stack=0x564c6f91c0c0 is io blocked 2026-03-20T11:46:23.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.318+0000 7f4df2ffd640 20 link_request req_data=0x564c6f81f660 req_data->id=35, curl_handle=0x564c6f81eb70 2026-03-20T11:46:23.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f966520:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f966520:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f95a530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f966520:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f95a530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f966520:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f95a530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f95edc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 15 stack 0x564c6f953ef0 end 2026-03-20T11:46:23.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 20 run: stack=0x564c6f953ef0 is done 2026-03-20T11:46:23.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f95a530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 20 collect(): s=0x564c6f907e20 stack=0x564c6f953ef0 is complete 2026-03-20T11:46:23.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 10 trimming bilogs for bucket=:dipepu-9[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14]) markers=[,00000000002.13.6,,,,,,,,,], shards=11 2026-03-20T11:46:23.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f947990:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 15 stack 0x564c6f94b500 end 2026-03-20T11:46:23.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 20 run: stack=0x564c6f94b500 is done 2026-03-20T11:46:23.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f821a30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 10 trimming bilog shard 1 of :dipepu-9[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14]) at marker 00000000002.13.6 2026-03-20T11:46:23.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 20 collect(): s=0x564c6f8132d0 stack=0x564c6f94b500 is complete 2026-03-20T11:46:23.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 10 trimming bilogs for bucket=:dipepu-38[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43]) markers=[,,,,,,,,00000000001.38.6,,], shards=11 2026-03-20T11:46:23.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f4a5950:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f95edc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 10 trimming bilog shard 8 of :dipepu-38[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43]) at marker 00000000001.38.6 2026-03-20T11:46:23.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f4a5950:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14.1 2026-03-20T11:46:23.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 20 run: stack=0x564c6f953ef0 is io blocked 2026-03-20T11:46:23.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f95f540:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f95f540:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43.8 2026-03-20T11:46:23.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.349+0000 7f4e96f52900 20 run: stack=0x564c6f94b500 is io blocked 2026-03-20T11:46:23.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f95f540:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f95f540:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 15 stack 0x564c6f94b500 end 2026-03-20T11:46:23.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 run: stack=0x564c6f94b500 is done 2026-03-20T11:46:23.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f95edc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 trim: trimmed bucket instance dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:23.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 15 stack 0x564c6f8132d0 end 2026-03-20T11:46:23.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is done 2026-03-20T11:46:23.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 4 starting trim on bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:23.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f95f540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f95f540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 enqueued request req=0x564c6f8130b0 2026-03-20T11:46:23.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 req: 0x564c6f8130b0 2026-03-20T11:46:23.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is io blocked 2026-03-20T11:46:23.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e507f8640 20 dequeued request req=0x564c6f8130b0 2026-03-20T11:46:23.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e507f8640 20 RGWWQ: empty 2026-03-20T11:46:23.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e507f8640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 : miss 2026-03-20T11:46:23.348 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e507f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e507f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f4a5950:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f4a5950:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 15 stack 0x564c6f953ef0 end 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e507f8640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 info.flags=0x17 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e507f8640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 to cache LRU end 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 run: stack=0x564c6f953ef0 is done 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e507f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e507f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f821a30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 trim: trimmed bucket instance dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 15 stack 0x564c6f907e20 end 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is done 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e507f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 4 starting trim on bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f81d3a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f81d3a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 enqueued request req=0x564c6f4a1710 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 req: 0x564c6f4a1710 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is io blocked 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e507f8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-11 : miss 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e507f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e50ff9640 20 dequeued request req=0x564c6f4a1710 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e50ff9640 20 RGWWQ: empty 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e50ff9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 : miss 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.351+0000 7f4e50ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e50ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e50ff9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 info.flags=0x17 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e50ff9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 to cache LRU end 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e50ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e50ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e50ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e50ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-12 : miss 2026-03-20T11:46:23.349 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e50ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e50ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e507f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e50ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-12 info.flags=0x0 2026-03-20T11:46:23.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e50ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-12 to cache LRU end 2026-03-20T11:46:23.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e50ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-12 : miss 2026-03-20T11:46:23.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e507f8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-11 info.flags=0x0 2026-03-20T11:46:23.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e50ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e507f8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-11 to cache LRU end 2026-03-20T11:46:23.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e507f8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-11 : miss 2026-03-20T11:46:23.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e507f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e50ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e50ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-12 info.flags=0x0 2026-03-20T11:46:23.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e50ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-12 to cache LRU end 2026-03-20T11:46:23.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e507f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e507f8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-11 info.flags=0x0 2026-03-20T11:46:23.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e507f8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-11 to cache LRU end 2026-03-20T11:46:23.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e507f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16): adding source pipe: {s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e50ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17): adding source pipe: {s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e50ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17): adding dest pipe: {s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e50ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e50ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.352+0000 7f4e50ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:23.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f81d3a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f81d3a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f81d3a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f81d3a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e507f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16): adding dest pipe: {s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.350 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f95edc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e507f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e507f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e507f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 5e444cfac388fd7cb1ea3fcb4e57f4cb8ecc2900a252365bf80ef38740fc6b0a 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:5e444cfac388fd7cb1ea3fcb4e57f4cb8ecc2900a252365bf80ef38740fc6b0a 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 0b5dd673619e31cf789e61e432efe0fb1678db4fa676f9a01e89e39ac240b7bb 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=0b5dd673619e31cf789e61e432efe0fb1678db4fa676f9a01e89e39ac240b7bb 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=36, curl_handle=0x564c6f911bf0 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 run: stack=0x564c6f953ef0 is io blocked 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f95f540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f95f540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f95f540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f95f540:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f966520:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f964810:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f964810:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4df2ffd640 20 link_request req_data=0x564c6f912300 req_data->id=36, curl_handle=0x564c6f911bf0 2026-03-20T11:46:23.351 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = f1ad09299fe79b08f2592569f6ba6155c9037a83aea3c67eea76520b70ea3044 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:f1ad09299fe79b08f2592569f6ba6155c9037a83aea3c67eea76520b70ea3044 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 5d6639d46938c21d668fdcb706caf25c55b168f7ae9c252544960108c75e5590 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5d6639d46938c21d668fdcb706caf25c55b168f7ae9c252544960108c75e5590 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=37, curl_handle=0x564c6f823420 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4e96f52900 20 run: stack=0x564c6f94b500 is io blocked 2026-03-20T11:46:23.352 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.353+0000 7f4df2ffd640 20 link_request req_data=0x564c6f918e60 req_data->id=37, curl_handle=0x564c6f823420 2026-03-20T11:46:23.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f965000:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f965000:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f94d7e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f965000:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f94d7e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f965000:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f94d7e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f9113b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 15 stack 0x564c6f821700 end 2026-03-20T11:46:23.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is done 2026-03-20T11:46:23.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f94d7e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 collect(): s=0x564c6f4a7b60 stack=0x564c6f821700 is complete 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 10 trimming bilogs for bucket=:dipepu-15[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20]) markers=[,,,,,,,,00000000007.20.6,,], shards=11 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f90ba50:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 15 stack 0x564c6f91c0c0 end 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 run: stack=0x564c6f91c0c0 is done 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f9113b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 10 trimming bilog shard 8 of :dipepu-15[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20]) at marker 00000000007.20.6 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f95b4c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 collect(): s=0x564c6f4a77d0 stack=0x564c6f91c0c0 is complete 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 10 trimming bilogs for bucket=:dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) markers=[,,,,,00000000003.29.10,,,00000000003.11.6,00000000011.37.6,], shards=11 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f4a5950:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f90ba50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 10 trimming bilog shard 5 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000003.29.10 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 10 trimming bilog shard 8 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000003.11.6 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 10 trimming bilog shard 9 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000011.37.6 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f4a5950:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20.8 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is io blocked 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f960a30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f94d7e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f90a760:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f960a30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.5 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 run: stack=0x564c6f91c0c0 is io blocked 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f94d7e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.8 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is io blocked 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f90a760:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.9 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.361+0000 7f4e96f52900 20 run: stack=0x564c6f9106c0 is io blocked 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f960a30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f960a30:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 15 stack 0x564c6f91c0c0 end 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 run: stack=0x564c6f91c0c0 is done 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f90ba50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f4a5950:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f4a5950:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 15 stack 0x564c6f821700 end 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is done 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f9113b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 trim: trimmed bucket instance dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 15 stack 0x564c6f4a7b60 end 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is done 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 4 starting trim on bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f94ab50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f94ab50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 enqueued request req=0x564c6f94c560 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 req: 0x564c6f94c560 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is io blocked 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e4f7f6640 20 dequeued request req=0x564c6f94c560 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e4f7f6640 20 RGWWQ: empty 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e4f7f6640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 : miss 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e4f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f90a760:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f90a760:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 15 stack 0x564c6f9106c0 end 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 run: stack=0x564c6f9106c0 is done 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f90ba50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f94d7e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f94d7e0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 15 stack 0x564c6f94bd80 end 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is done 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f90ba50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f95b4c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 trim: trimmed bucket instance dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 15 stack 0x564c6f4a77d0 end 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is done 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f95b4c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 4 starting trim on bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f821840:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f821840:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e4f7f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e4f7f6640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 info.flags=0x17 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e4f7f6640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 to cache LRU end 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e4f7f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e4f7f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 enqueued request req=0x564c6f90ee00 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 req: 0x564c6f90ee00 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is io blocked 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e4fff7640 20 dequeued request req=0x564c6f90ee00 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e4fff7640 20 RGWWQ: empty 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e4fff7640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 : miss 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e4fff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.363+0000 7f4e4f7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4f7f6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-40 : miss 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4fff7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4fff7640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 info.flags=0x17 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4fff7640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 to cache LRU end 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4fff7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4fff7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4fff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:23.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4fff7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-18 : miss 2026-03-20T11:46:23.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4fff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4f7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4f7f6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-40 info.flags=0x0 2026-03-20T11:46:23.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4f7f6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-40 to cache LRU end 2026-03-20T11:46:23.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4f7f6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-40 : miss 2026-03-20T11:46:23.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4fff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4fff7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-18 info.flags=0x0 2026-03-20T11:46:23.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4fff7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-18 to cache LRU end 2026-03-20T11:46:23.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4fff7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-18 : miss 2026-03-20T11:46:23.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4fff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4f7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4f7f6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-40 info.flags=0x0 2026-03-20T11:46:23.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4f7f6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-40 to cache LRU end 2026-03-20T11:46:23.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1): adding source pipe: {s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1): adding dest pipe: {s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4f7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:23.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4fff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.362 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4fff7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-18 info.flags=0x0 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4fff7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-18 to cache LRU end 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4fff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23): adding source pipe: {s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4fff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23): adding dest pipe: {s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4fff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.364+0000 7f4e4fff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e4fff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f94ab50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f94ab50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f94ab50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f94ab50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f821840:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f9113b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f821840:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f94e350:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f821840:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f94e350:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 0e8f522d4f4e59d0c34a15b7e57aeba29ec8716cd891d79d8ddae8c812638b98 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:0e8f522d4f4e59d0c34a15b7e57aeba29ec8716cd891d79d8ddae8c812638b98 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = a141d447496ae995e45dcf1003371488255ff416e6d7c6ad2d4a529757e2ce9a 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a141d447496ae995e45dcf1003371488255ff416e6d7c6ad2d4a529757e2ce9a 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=38, curl_handle=0x564c6f81eb70 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is io blocked 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f821840:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f95b4c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f95d590:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f95b4c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f952340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f952340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4df2ffd640 20 link_request req_data=0x564c6f81f660 req_data->id=38, curl_handle=0x564c6f81eb70 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 732c93a93c7b237ed86300b6edc57089099f4f1610d3f8fe08aa435c73ec7aa9 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:732c93a93c7b237ed86300b6edc57089099f4f1610d3f8fe08aa435c73ec7aa9 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 5f334f6c573be3d618bd4d676f79d85378de6b2b6e60ea8400dfe62e1310783e 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5f334f6c573be3d618bd4d676f79d85378de6b2b6e60ea8400dfe62e1310783e 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=39, curl_handle=0x564c6f915d30 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4e96f52900 20 run: stack=0x564c6f9106c0 is io blocked 2026-03-20T11:46:23.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.365+0000 7f4df2ffd640 20 link_request req_data=0x564c6f954490 req_data->id=39, curl_handle=0x564c6f915d30 2026-03-20T11:46:23.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f95edc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.393 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 15 stack 0x564c6f953ef0 end 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 run: stack=0x564c6f953ef0 is done 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 collect(): s=0x564c6f907e20 stack=0x564c6f953ef0 is complete 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 10 trimming bilogs for bucket=:dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) markers=[00000000014.44.6,00000000008.52.6,00000000008.40.6,00000000008.21.6,00000000012.26.6,00000000010.26.6,00000000008.35.6,00000000010.32.6,00000000004.38.6,00000000008.22.6,00000000008.23.6], shards=11 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f95edc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 10 trimming bilog shard 0 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000014.44.6 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 10 trimming bilog shard 1 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.52.6 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 10 trimming bilog shard 2 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.40.6 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 10 trimming bilog shard 3 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.21.6 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 10 trimming bilog shard 4 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000012.26.6 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 10 trimming bilog shard 5 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000010.26.6 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 10 trimming bilog shard 6 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.35.6 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 10 trimming bilog shard 7 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000010.32.6 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 10 trimming bilog shard 8 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000004.38.6 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 10 trimming bilog shard 9 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.22.6 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 10 trimming bilog shard 10 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.23.6 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f4a5950:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f9694a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f957a70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f96b9b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f94b640:op=0x564c6f94f5d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f9541b0:op=0x564c6f90a790:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f822cd0:op=0x564c6f968640:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f917ae0:op=0x564c6f94d7e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f96c1b0:op=0x564c6f81ccf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f96c330:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f96cb30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f4a5950:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.0 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 run: stack=0x564c6f953ef0 is io blocked 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f9694a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.1 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is io blocked 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f957a70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.2 2026-03-20T11:46:23.394 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 run: stack=0x564c6f91c0c0 is io blocked 2026-03-20T11:46:23.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f96b9b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.3 2026-03-20T11:46:23.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 run: stack=0x564c6f8210c0 is io blocked 2026-03-20T11:46:23.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f94b640:op=0x564c6f94f5d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.4 2026-03-20T11:46:23.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 run: stack=0x564c6f94b640 is io blocked 2026-03-20T11:46:23.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f9541b0:op=0x564c6f90a790:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.5 2026-03-20T11:46:23.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 run: stack=0x564c6f9541b0 is io blocked 2026-03-20T11:46:23.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f822cd0:op=0x564c6f968640:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.6 2026-03-20T11:46:23.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 run: stack=0x564c6f822cd0 is io blocked 2026-03-20T11:46:23.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f917ae0:op=0x564c6f94d7e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.7 2026-03-20T11:46:23.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 run: stack=0x564c6f917ae0 is io blocked 2026-03-20T11:46:23.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f96c1b0:op=0x564c6f81ccf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.8 2026-03-20T11:46:23.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 run: stack=0x564c6f96c1b0 is io blocked 2026-03-20T11:46:23.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f96c330:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.9 2026-03-20T11:46:23.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 run: stack=0x564c6f94c000 is io blocked 2026-03-20T11:46:23.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f96cb30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.10 2026-03-20T11:46:23.395 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.396+0000 7f4e96f52900 20 run: stack=0x564c6f91b5f0 is io blocked 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.397+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f964810:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f964810:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f964810:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f964810:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f966520:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 15 stack 0x564c6f94b500 end 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 run: stack=0x564c6f94b500 is done 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 collect(): s=0x564c6f8132d0 stack=0x564c6f94b500 is complete 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 10 trimming bilogs for bucket=:dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) markers=[00000000014.52.6,00000000008.34.6,00000000008.50.6,00000000008.32.6,00000000012.28.6,00000000010.31.6,00000000008.21.6,00000000010.27.6,00000000004.27.6,00000000008.22.6,00000000008.19.6], shards=11 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f965000:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 10 trimming bilog shard 0 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000014.52.6 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 10 trimming bilog shard 1 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.34.6 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 10 trimming bilog shard 2 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.50.6 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 10 trimming bilog shard 3 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.32.6 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 10 trimming bilog shard 4 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000012.28.6 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 10 trimming bilog shard 5 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000010.31.6 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 10 trimming bilog shard 6 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.21.6 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 10 trimming bilog shard 7 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000010.27.6 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 10 trimming bilog shard 8 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000004.27.6 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 10 trimming bilog shard 9 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.22.6 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 10 trimming bilog shard 10 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.19.6 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f96c330:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f96c330:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 15 stack 0x564c6f94c000 end 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 run: stack=0x564c6f94c000 is done 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f95f540:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f94a660:op=0x564c6f98af70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f986f20:op=0x564c6f98a280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.396 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f94e990:op=0x564c6f9885a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f91c260:op=0x564c6f989440:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f9866d0:op=0x564c6f98e9f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f947e40:op=0x564c6f98f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f98ba50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f93f400:op=0x564c6f98c250:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f96a020:op=0x564c6f984940:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f967710:op=0x564c6f985140:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f96b9b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f96b9b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 15 stack 0x564c6f8210c0 end 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 run: stack=0x564c6f8210c0 is done 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f917ae0:op=0x564c6f94d7e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f917ae0:op=0x564c6f94d7e0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 15 stack 0x564c6f917ae0 end 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 run: stack=0x564c6f917ae0 is done 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f95edc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f95f540:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.0 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 run: stack=0x564c6f94b500 is io blocked 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f94a660:op=0x564c6f98af70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.1 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 run: stack=0x564c6f94a660 is io blocked 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f986f20:op=0x564c6f98a280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.2 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 run: stack=0x564c6f986f20 is io blocked 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f94e990:op=0x564c6f9885a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.3 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 run: stack=0x564c6f94e990 is io blocked 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f91c260:op=0x564c6f989440:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.4 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 run: stack=0x564c6f91c260 is io blocked 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f9866d0:op=0x564c6f98e9f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.5 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 run: stack=0x564c6f9866d0 is io blocked 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f947e40:op=0x564c6f98f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.6 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 run: stack=0x564c6f947e40 is io blocked 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f98ba50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.7 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 run: stack=0x564c6f4a1590 is io blocked 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f93f400:op=0x564c6f98c250:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.8 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 run: stack=0x564c6f93f400 is io blocked 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f96a020:op=0x564c6f984940:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.9 2026-03-20T11:46:23.397 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 run: stack=0x564c6f96a020 is io blocked 2026-03-20T11:46:23.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f967710:op=0x564c6f985140:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.10 2026-03-20T11:46:23.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 run: stack=0x564c6f967710 is io blocked 2026-03-20T11:46:23.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f95edc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f95edc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.398 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f94b640:op=0x564c6f94f5d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f94b640:op=0x564c6f94f5d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 15 stack 0x564c6f94b640 end 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 run: stack=0x564c6f94b640 is done 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f95edc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.398+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 cr:s=0x564c6f9541b0:op=0x564c6f90a790:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 cr:s=0x564c6f9541b0:op=0x564c6f90a790:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 15 stack 0x564c6f9541b0 end 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 run: stack=0x564c6f9541b0 is done 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f95edc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 cr:s=0x564c6f822cd0:op=0x564c6f968640:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 cr:s=0x564c6f822cd0:op=0x564c6f968640:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 15 stack 0x564c6f822cd0 end 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 run: stack=0x564c6f822cd0 is done 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f4a5950:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f4a5950:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 15 stack 0x564c6f953ef0 end 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 run: stack=0x564c6f953ef0 is done 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f957a70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f957a70:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 15 stack 0x564c6f91c0c0 end 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 run: stack=0x564c6f91c0c0 is done 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f96cb30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f96cb30:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 15 stack 0x564c6f91b5f0 end 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 run: stack=0x564c6f91b5f0 is done 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 cr:s=0x564c6f96c1b0:op=0x564c6f81ccf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 cr:s=0x564c6f96c1b0:op=0x564c6f81ccf0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 15 stack 0x564c6f96c1b0 end 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 run: stack=0x564c6f96c1b0 is done 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f95edc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f95edc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f95edc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f95edc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f95edc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.399+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.399 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.400+0000 7f4e96f52900 20 cr:s=0x564c6f96a020:op=0x564c6f984940:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f96a020:op=0x564c6f984940:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 15 stack 0x564c6f96a020 end 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 run: stack=0x564c6f96a020 is done 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f965000:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f95f540:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f94b500:op=0x564c6f95f540:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 15 stack 0x564c6f94b500 end 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 run: stack=0x564c6f94b500 is done 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f986f20:op=0x564c6f98a280:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f986f20:op=0x564c6f98a280:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 15 stack 0x564c6f986f20 end 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 run: stack=0x564c6f986f20 is done 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f965000:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f965000:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f967710:op=0x564c6f985140:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f967710:op=0x564c6f985140:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 15 stack 0x564c6f967710 end 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 run: stack=0x564c6f967710 is done 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f965000:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f9866d0:op=0x564c6f98e9f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f9866d0:op=0x564c6f98e9f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 15 stack 0x564c6f9866d0 end 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 run: stack=0x564c6f9866d0 is done 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f965000:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f98ba50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f98ba50:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 15 stack 0x564c6f4a1590 end 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 run: stack=0x564c6f4a1590 is done 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f9694a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f821700:op=0x564c6f9694a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 15 stack 0x564c6f821700 end 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 run: stack=0x564c6f821700 is done 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f94e990:op=0x564c6f9885a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f94e990:op=0x564c6f9885a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 15 stack 0x564c6f94e990 end 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 run: stack=0x564c6f94e990 is done 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f91c260:op=0x564c6f989440:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f91c260:op=0x564c6f989440:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 15 stack 0x564c6f91c260 end 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 run: stack=0x564c6f91c260 is done 2026-03-20T11:46:23.400 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f947e40:op=0x564c6f98f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f947e40:op=0x564c6f98f1f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 15 stack 0x564c6f947e40 end 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 run: stack=0x564c6f947e40 is done 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f93f400:op=0x564c6f98c250:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f93f400:op=0x564c6f98c250:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 15 stack 0x564c6f93f400 end 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 run: stack=0x564c6f93f400 is done 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f965000:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f95edc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f965000:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f907e20:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 trim: trimmed bucket instance dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 15 stack 0x564c6f907e20 end 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 run: stack=0x564c6f907e20 is done 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f965000:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f965000:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f984940:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 4 starting trim on bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f965000:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 enqueued request req=0x564c6f9657e0 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 req: 0x564c6f9657e0 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 run: stack=0x564c6f91b5f0 is io blocked 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e4eff5640 20 dequeued request req=0x564c6f9657e0 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e4eff5640 20 RGWWQ: empty 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e4eff5640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 : miss 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e4eff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f94a660:op=0x564c6f98af70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f94a660:op=0x564c6f98af70:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 15 stack 0x564c6f94a660 end 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 run: stack=0x564c6f94a660 is done 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f965000:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f8132d0:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 trim: trimmed bucket instance dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 15 stack 0x564c6f8132d0 end 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 run: stack=0x564c6f8132d0 is done 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 4 starting trim on bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f4a5950:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f4a5950:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 enqueued request req=0x564c6f90b800 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 req: 0x564c6f90b800 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e96f52900 20 run: stack=0x564c6f4a1590 is io blocked 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e4e7f4640 20 dequeued request req=0x564c6f90b800 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e4e7f4640 20 RGWWQ: empty 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e4e7f4640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 : miss 2026-03-20T11:46:23.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e4e7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e4eff5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e4eff5640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 info.flags=0x17 2026-03-20T11:46:23.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e4eff5640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 to cache LRU end 2026-03-20T11:46:23.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e4eff5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e4eff5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e4eff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:23.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e4eff5640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-50 : miss 2026-03-20T11:46:23.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e4eff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e4e7f4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e4e7f4640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 info.flags=0x17 2026-03-20T11:46:23.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e4e7f4640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 to cache LRU end 2026-03-20T11:46:23.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e4e7f4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e4e7f4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e4e7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:23.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e4e7f4640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-24 : miss 2026-03-20T11:46:23.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.401+0000 7f4e4e7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e4eff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e4eff5640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-50 info.flags=0x0 2026-03-20T11:46:23.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e4eff5640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-50 to cache LRU end 2026-03-20T11:46:23.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e4e7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e4eff5640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-50 : miss 2026-03-20T11:46:23.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e4e7f4640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-24 info.flags=0x0 2026-03-20T11:46:23.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e4e7f4640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-24 to cache LRU end 2026-03-20T11:46:23.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e4eff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e4e7f4640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-24 : miss 2026-03-20T11:46:23.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e4e7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.402 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:23.402+0000 7fd024296640 -1 req 975399144695781681 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:23.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e4e7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e4e7f4640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-24 info.flags=0x0 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e4e7f4640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-24 to cache LRU end 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e4eff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e4eff5640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-50 info.flags=0x0 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e4eff5640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-50 to cache LRU end 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e4e7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29): adding source pipe: {s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e4eff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10): adding source pipe: {s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e4e7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29): adding dest pipe: {s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e4eff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10): adding dest pipe: {s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e4e7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e4eff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e4e7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e4eff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e4eff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e4e7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f4a5950:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f4a5950:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f4a5950:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f984940:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f4a5950:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f95edc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f984940:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 run: stack=0x564c6f91b5f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f964810:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 run: stack=0x564c6f4a1590 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.403 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 884a8f07a3e0b7793b746984d1a43672b5a904298b3e5e591e303fcb0094a4f0 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:884a8f07a3e0b7793b746984d1a43672b5a904298b3e5e591e303fcb0094a4f0 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 2e4e3e7ccabb51b60fe99fce963baad56fb5ac89e27aebe34a7745f80a67e158 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2e4e3e7ccabb51b60fe99fce963baad56fb5ac89e27aebe34a7745f80a67e158 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=40, curl_handle=0x564c6f823420 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 run: stack=0x564c6f91c0c0 is io blocked 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f987830:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f987830:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4df2ffd640 20 link_request req_data=0x564c6f918e60 req_data->id=40, curl_handle=0x564c6f823420 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 993db31c19d3d2fcb89119349cfe0365f48beda126667ff16b96c37a1a1aed4f 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:993db31c19d3d2fcb89119349cfe0365f48beda126667ff16b96c37a1a1aed4f 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 5a48baa728a894d941fe0047948e76726483e31b41f4485f8ed42d778b137df4 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5a48baa728a894d941fe0047948e76726483e31b41f4485f8ed42d778b137df4 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.404 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.405 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.405 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=41, curl_handle=0x564c6f911bf0 2026-03-20T11:46:23.405 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4e96f52900 20 run: stack=0x564c6f953ef0 is io blocked 2026-03-20T11:46:23.405 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.402+0000 7f4df2ffd640 20 link_request req_data=0x564c6f912300 req_data->id=41, curl_handle=0x564c6f911bf0 2026-03-20T11:46:23.405 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f94e350:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f94e350:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f952340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f94e350:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f952340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f94e350:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f952340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f9113b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 15 stack 0x564c6f94bd80 end 2026-03-20T11:46:23.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is done 2026-03-20T11:46:23.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f952340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 20 collect(): s=0x564c6f4a7b60 stack=0x564c6f94bd80 is complete 2026-03-20T11:46:23.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 10 trimming bilogs for bucket=:dipepu-40[4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1]) markers=[,,,,,00000000001.103.6,,,,,], shards=11 2026-03-20T11:46:23.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f95d590:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 15 stack 0x564c6f9106c0 end 2026-03-20T11:46:23.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 20 run: stack=0x564c6f9106c0 is done 2026-03-20T11:46:23.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f95dbb0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 10 trimming bilog shard 5 of :dipepu-40[4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1]) at marker 00000000001.103.6 2026-03-20T11:46:23.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f95b4c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 20 collect(): s=0x564c6f4a77d0 stack=0x564c6f9106c0 is complete 2026-03-20T11:46:23.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 10 trimming bilogs for bucket=:dipepu-18[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23]) markers=[,,,,,,,,00000000005.51.12,,], shards=11 2026-03-20T11:46:23.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f94f5d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f952340:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.406 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 10 trimming bilog shard 8 of :dipepu-18[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23]) at marker 00000000005.51.12 2026-03-20T11:46:23.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f94f5d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1.5 2026-03-20T11:46:23.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is io blocked 2026-03-20T11:46:23.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f95a530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f95a530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23.8 2026-03-20T11:46:23.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.409+0000 7f4e96f52900 20 run: stack=0x564c6f9106c0 is io blocked 2026-03-20T11:46:23.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f95a530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f95a530:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 15 stack 0x564c6f9106c0 end 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 run: stack=0x564c6f9106c0 is done 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f952340:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f94f5d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f94f5d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 15 stack 0x564c6f94bd80 end 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is done 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f95b4c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 trim: trimmed bucket instance dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 15 stack 0x564c6f4a77d0 end 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is done 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f95dbb0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 trim: trimmed bucket instance dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 15 stack 0x564c6f4a7b60 end 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is done 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f94d7e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 4 starting trim on bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 4 starting trim on bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 enqueued request req=0x564c6f908e00 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 req: 0x564c6f908e00 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is io blocked 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f94f5d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f94f5d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 enqueued request req=0x564c6f4a1710 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 req: 0x564c6f908e00 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 req: 0x564c6f4a1710 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is io blocked 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e4dff3640 20 dequeued request req=0x564c6f908e00 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e4dff3640 20 RGWWQ: 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e4dff3640 20 req: 0x564c6f4a1710 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e4dff3640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 : miss 2026-03-20T11:46:23.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e4dff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e4d7f2640 20 dequeued request req=0x564c6f4a1710 2026-03-20T11:46:23.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e4d7f2640 20 RGWWQ: empty 2026-03-20T11:46:23.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e4d7f2640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 : miss 2026-03-20T11:46:23.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.410+0000 7f4e4d7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4dff3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4dff3640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 info.flags=0x17 2026-03-20T11:46:23.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4dff3640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 to cache LRU end 2026-03-20T11:46:23.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4dff3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4dff3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4dff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:46:23.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4dff3640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-56 : miss 2026-03-20T11:46:23.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4dff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4d7f2640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4d7f2640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 info.flags=0x17 2026-03-20T11:46:23.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4d7f2640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 to cache LRU end 2026-03-20T11:46:23.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4d7f2640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4d7f2640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4d7f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:46:23.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4d7f2640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-27 : miss 2026-03-20T11:46:23.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4d7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4d7f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4d7f2640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-27 info.flags=0x0 2026-03-20T11:46:23.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4d7f2640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-27 to cache LRU end 2026-03-20T11:46:23.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4d7f2640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-27 : miss 2026-03-20T11:46:23.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4d7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4dff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4dff3640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-56 info.flags=0x0 2026-03-20T11:46:23.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4dff3640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-56 to cache LRU end 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4dff3640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-56 : miss 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4dff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4d7f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4d7f2640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-27 info.flags=0x0 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4d7f2640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-27 to cache LRU end 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4d7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32): adding source pipe: {s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4d7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32): adding dest pipe: {s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4d7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4d7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4d7f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4dff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4dff3640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-56 info.flags=0x0 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4dff3640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-56 to cache LRU end 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f94f5d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16): adding source pipe: {s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f94f5d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f94f5d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f94f5d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16): adding dest pipe: {s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f9113b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f969ee0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f969ee0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e4dff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 3185fadec81a6ef988b1a4452246b934e4ec0f03709362ba7ea928c77e48489d 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:3185fadec81a6ef988b1a4452246b934e4ec0f03709362ba7ea928c77e48489d 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.411+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 696ba9981c69080d638d15f01e70498ba2c2bfda3b1422bd26361a03ad6e1b2d 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=696ba9981c69080d638d15f01e70498ba2c2bfda3b1422bd26361a03ad6e1b2d 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.410 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=42, curl_handle=0x564c6f915d30 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is io blocked 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f94d7e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f909ae0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f94d7e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f986060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f986060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4df2ffd640 20 link_request req_data=0x564c6f954490 req_data->id=42, curl_handle=0x564c6f915d30 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = f1a630beb9c16989a1b2f3fdf3226215d6f44479be903ed63db6960f6ccbd3c7 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:f1a630beb9c16989a1b2f3fdf3226215d6f44479be903ed63db6960f6ccbd3c7 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = c70ddd0baaafb59367552d44282ee23e49cc50880b59f6749ee458c0ff217adc 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c70ddd0baaafb59367552d44282ee23e49cc50880b59f6749ee458c0ff217adc 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=43, curl_handle=0x564c6f81eb70 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4e96f52900 20 run: stack=0x564c6f9106c0 is io blocked 2026-03-20T11:46:23.411 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.412+0000 7f4df2ffd640 20 link_request req_data=0x564c6f81f660 req_data->id=43, curl_handle=0x564c6f81eb70 2026-03-20T11:46:23.411 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:23.411+0000 7fd00ca67640 -1 req 1144416840704219223 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:23.411 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:23.411+0000 7fd0512f0640 -1 req 15570668986663504828 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:23.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f987830:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f987830:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:23.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f964810:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 15 stack 0x564c6f953ef0 end 2026-03-20T11:46:23.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 run: stack=0x564c6f953ef0 is done 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 collect(): s=0x564c6f4a1590 stack=0x564c6f953ef0 is complete 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 15 stack 0x564c6f4a1590 end 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 stack->operate() returned ret=-22 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 run: stack=0x564c6f4a1590 is done 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f95edc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 15 stack 0x564c6f91c0c0 end 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 run: stack=0x564c6f91c0c0 is done 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f984940:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 collect(): s=0x564c6f91b5f0 stack=0x564c6f91c0c0 is complete 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f984940:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 15 stack 0x564c6f91b5f0 end 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 stack->operate() returned ret=-22 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 run: stack=0x564c6f91b5f0 is done 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 4 starting trim on bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f9696f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 4 starting trim on bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f9696f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 enqueued request req=0x564c6f90b800 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 req: 0x564c6f90b800 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 run: stack=0x564c6f4a1590 is io blocked 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 enqueued request req=0x564c6f9657e0 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 req: 0x564c6f90b800 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 req: 0x564c6f9657e0 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e96f52900 20 run: stack=0x564c6f91b5f0 is io blocked 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e4cff1640 20 dequeued request req=0x564c6f90b800 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e4cff1640 20 RGWWQ: 2026-03-20T11:46:23.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e4cff1640 20 req: 0x564c6f9657e0 2026-03-20T11:46:23.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e4c7f0640 20 dequeued request req=0x564c6f9657e0 2026-03-20T11:46:23.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e4c7f0640 20 RGWWQ: empty 2026-03-20T11:46:23.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e4cff1640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 : miss 2026-03-20T11:46:23.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e4c7f0640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 : miss 2026-03-20T11:46:23.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e4c7f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e4cff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e4c7f0640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e4c7f0640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 info.flags=0x17 2026-03-20T11:46:23.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e4c7f0640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 to cache LRU end 2026-03-20T11:46:23.443 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e4c7f0640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e4c7f0640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e4c7f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e4c7f0640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-17 : miss 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.444+0000 7f4e4c7f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4cff1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=314 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4cff1640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 info.flags=0x17 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4cff1640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 to cache LRU end 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4cff1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4cff1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4cff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4cff1640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-44 : miss 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4cff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4cff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4cff1640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-44 info.flags=0x0 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4cff1640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-44 to cache LRU end 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4cff1640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-44 : miss 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4cff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4c7f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4c7f0640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-17 info.flags=0x0 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4c7f0640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-17 to cache LRU end 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4c7f0640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-17 : miss 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4c7f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4c7f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4c7f0640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-17 info.flags=0x0 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4c7f0640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-17 to cache LRU end 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4c7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22): adding source pipe: {s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4c7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22): adding dest pipe: {s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4c7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4c7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4c7f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4cff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4cff1640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-44 info.flags=0x0 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4cff1640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-44 to cache LRU end 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4cff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4): adding source pipe: {s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4cff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4): adding dest pipe: {s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f95edc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 run: stack=0x564c6f91b5f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4cff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4cff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e4cff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 22e489161f7fdafa05ee95c0a8cbcddbaa6bcff543b3ddbb15df74fe6f1ba705 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:22e489161f7fdafa05ee95c0a8cbcddbaa6bcff543b3ddbb15df74fe6f1ba705 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = fc8856242313cd776f216fcb1513e676afb5448655604cf82a017b807109b61f 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=fc8856242313cd776f216fcb1513e676afb5448655604cf82a017b807109b61f 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=44, curl_handle=0x564c6f911bf0 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 run: stack=0x564c6f91c0c0 is io blocked 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f9696f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f9696f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f9696f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f9696f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f964760:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 run: stack=0x564c6f4a1590 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f9696f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f9696f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4df2ffd640 20 link_request req_data=0x564c6f912300 req_data->id=44, curl_handle=0x564c6f911bf0 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = bd1103a31399d7f2933126523e2f3a576d090e2b81929e2896c7fd3d69558aab 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:bd1103a31399d7f2933126523e2f3a576d090e2b81929e2896c7fd3d69558aab 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 484a7adc9f0643146685d9b8ff884df6930eb81320a4fc8f76a9baca692d0848 2026-03-20T11:46:23.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=484a7adc9f0643146685d9b8ff884df6930eb81320a4fc8f76a9baca692d0848 2026-03-20T11:46:23.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=45, curl_handle=0x564c6f823420 2026-03-20T11:46:23.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4e96f52900 20 run: stack=0x564c6f953ef0 is io blocked 2026-03-20T11:46:23.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.445+0000 7f4df2ffd640 20 link_request req_data=0x564c6f918e60 req_data->id=45, curl_handle=0x564c6f823420 2026-03-20T11:46:23.446 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:23.445+0000 7fd077b3d640 -1 req 6970377595368707043 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f969ee0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f969ee0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f9113b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 15 stack 0x564c6f94bd80 end 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is done 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f986060:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f986060:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 collect(): s=0x564c6f4a7b60 stack=0x564c6f94bd80 is complete 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 15 stack 0x564c6f4a7b60 end 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 stack->operate() returned ret=-22 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is done 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f909ae0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 15 stack 0x564c6f9106c0 end 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 run: stack=0x564c6f9106c0 is done 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f94d7e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 collect(): s=0x564c6f4a77d0 stack=0x564c6f9106c0 is complete 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f94d7e0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 15 stack 0x564c6f4a77d0 end 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 stack->operate() returned ret=-22 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is done 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 4 starting trim on bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f94d7e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 4 starting trim on bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:46:23.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 enqueued request req=0x564c6f94c560 2026-03-20T11:46:23.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 req: 0x564c6f94c560 2026-03-20T11:46:23.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is io blocked 2026-03-20T11:46:23.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f94ab50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f94ab50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 enqueued request req=0x564c6f4a1710 2026-03-20T11:46:23.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 req: 0x564c6f94c560 2026-03-20T11:46:23.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 req: 0x564c6f4a1710 2026-03-20T11:46:23.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is io blocked 2026-03-20T11:46:23.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e4bfef640 20 dequeued request req=0x564c6f94c560 2026-03-20T11:46:23.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e4bfef640 20 RGWWQ: 2026-03-20T11:46:23.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e4bfef640 20 req: 0x564c6f4a1710 2026-03-20T11:46:23.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e4bfef640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 : miss 2026-03-20T11:46:23.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e4bfef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e4b7ee640 20 dequeued request req=0x564c6f4a1710 2026-03-20T11:46:23.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e4b7ee640 20 RGWWQ: empty 2026-03-20T11:46:23.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e4b7ee640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 : miss 2026-03-20T11:46:23.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e4b7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e4bfef640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e4bfef640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 info.flags=0x17 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e4bfef640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 to cache LRU end 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e4bfef640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e4bfef640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e4bfef640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e4bfef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e4bfef640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-46 : miss 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e4b7ee640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e4bfef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e4b7ee640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 info.flags=0x17 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e4b7ee640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 to cache LRU end 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e4b7ee640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e4b7ee640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e4b7ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e4b7ee640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-37 : miss 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.453+0000 7f4e4b7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e4bfef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e4bfef640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-46 info.flags=0x0 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e4bfef640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-46 to cache LRU end 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e4bfef640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-46 : miss 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e4bfef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e4b7ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e4b7ee640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-37 info.flags=0x0 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e4b7ee640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-37 to cache LRU end 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e4b7ee640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-37 : miss 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e4b7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e4bfef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e4bfef640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-46 info.flags=0x0 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e4bfef640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-46 to cache LRU end 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e4bfef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6): adding source pipe: {s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e4bfef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6): adding dest pipe: {s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e4bfef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e4bfef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e4bfef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e4b7ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e4b7ee640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-37 info.flags=0x0 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e4b7ee640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-37 to cache LRU end 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e4b7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42): adding source pipe: {s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f914100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f9113b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f952340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f952340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 491967f34bbcc55066d21168ad7cc9cbebce61120c734b49f0c07fee4baebe7b 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:491967f34bbcc55066d21168ad7cc9cbebce61120c734b49f0c07fee4baebe7b 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 24321460d612d1d6213815ced5bae7febdbbcc3bd0fd66547858f17d96059c7b 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=24321460d612d1d6213815ced5bae7febdbbcc3bd0fd66547858f17d96059c7b 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=46, curl_handle=0x564c6f81eb70 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 run: stack=0x564c6f9106c0 is io blocked 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4df2ffd640 20 link_request req_data=0x564c6f81f660 req_data->id=46, curl_handle=0x564c6f81eb70 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e4b7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42): adding dest pipe: {s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e4b7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e4b7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e4b7ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f94ab50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f94ab50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f94ab50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f94ab50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f94d7e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f969ee0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f94d7e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f909ae0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f909ae0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = f05ca1b2aab41b0402ebcfe2221c4bbff98e565e705e3cb0d282ebe3bd58cb6a 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:f05ca1b2aab41b0402ebcfe2221c4bbff98e565e705e3cb0d282ebe3bd58cb6a 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 2abba2f6cffac17a43e6c090f5f424aaca47abfc5333ea34db38f656adc75406 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2abba2f6cffac17a43e6c090f5f424aaca47abfc5333ea34db38f656adc75406 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=47, curl_handle=0x564c6f915d30 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is io blocked 2026-03-20T11:46:23.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.454+0000 7f4df2ffd640 20 link_request req_data=0x564c6f954490 req_data->id=47, curl_handle=0x564c6f915d30 2026-03-20T11:46:23.454 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:23.454+0000 7fd04dae9640 -1 req 176426012074123071 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:23.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f9696f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f9696f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:23.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f964760:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e96f52900 15 stack 0x564c6f953ef0 end 2026-03-20T11:46:23.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e96f52900 20 run: stack=0x564c6f953ef0 is done 2026-03-20T11:46:23.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e96f52900 20 collect(): s=0x564c6f4a1590 stack=0x564c6f953ef0 is complete 2026-03-20T11:46:23.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e96f52900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:46:23.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e96f52900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:23.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:23.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e96f52900 15 stack 0x564c6f4a1590 end 2026-03-20T11:46:23.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e96f52900 20 stack->operate() returned ret=-22 2026-03-20T11:46:23.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e96f52900 20 run: stack=0x564c6f4a1590 is done 2026-03-20T11:46:23.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e96f52900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:23.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e96f52900 4 starting trim on bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:46:23.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f94ab50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f94ab50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e96f52900 20 enqueued request req=0x564c6f90b800 2026-03-20T11:46:23.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e96f52900 20 req: 0x564c6f90b800 2026-03-20T11:46:23.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e96f52900 20 run: stack=0x564c6f4a1590 is io blocked 2026-03-20T11:46:23.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e4afed640 20 dequeued request req=0x564c6f90b800 2026-03-20T11:46:23.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e4afed640 20 RGWWQ: empty 2026-03-20T11:46:23.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e4afed640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 : miss 2026-03-20T11:46:23.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e4afed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e4afed640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:46:23.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e4afed640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 info.flags=0x17 2026-03-20T11:46:23.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e4afed640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 to cache LRU end 2026-03-20T11:46:23.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e4afed640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e4afed640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e4afed640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:46:23.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e4afed640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-1 : miss 2026-03-20T11:46:23.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.487+0000 7f4e4afed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.488+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.488+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.488+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.488+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.488+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f95edc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.488+0000 7f4e96f52900 15 stack 0x564c6f91c0c0 end 2026-03-20T11:46:23.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.488+0000 7f4e96f52900 20 run: stack=0x564c6f91c0c0 is done 2026-03-20T11:46:23.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.488+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.488+0000 7f4e96f52900 20 collect(): s=0x564c6f91b5f0 stack=0x564c6f91c0c0 is complete 2026-03-20T11:46:23.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.488+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.488+0000 7f4e96f52900 10 trimming bilogs for bucket=:dipepu-17[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22]) markers=[,,,,,,,,00000000005.46.18,,], shards=11 2026-03-20T11:46:23.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.488+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f965000:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.488+0000 7f4e96f52900 10 trimming bilog shard 8 of :dipepu-17[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22]) at marker 00000000005.46.18 2026-03-20T11:46:23.485 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.488+0000 7f4e96f52900 20 run: stack=0x564c6f91b5f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.488+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f968a70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.488+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f968a70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.488+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22.8 2026-03-20T11:46:23.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.488+0000 7f4e96f52900 20 run: stack=0x564c6f91c0c0 is io blocked 2026-03-20T11:46:23.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.488+0000 7f4e4afed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.488+0000 7f4e4afed640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-1 info.flags=0x0 2026-03-20T11:46:23.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.488+0000 7f4e4afed640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-1 to cache LRU end 2026-03-20T11:46:23.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.488+0000 7f4e4afed640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-1 : miss 2026-03-20T11:46:23.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.488+0000 7f4e4afed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.488+0000 7f4e4afed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.488+0000 7f4e4afed640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-1 info.flags=0x0 2026-03-20T11:46:23.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.488+0000 7f4e4afed640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-1 to cache LRU end 2026-03-20T11:46:23.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e4afed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1): adding source pipe: {s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e4afed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1): adding dest pipe: {s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e4afed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e4afed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e4afed640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:46:23.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f94ab50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f94ab50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f94ab50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f94ab50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f95edc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 run: stack=0x564c6f4a1590 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = bb661e8cb1db1981de9bbb8a9dec0a9f73feea28c893473b04509ac213778107 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:bb661e8cb1db1981de9bbb8a9dec0a9f73feea28c893473b04509ac213778107 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 4fe551b6be96c12334901fa1bf111a4fcedbba7db249f9566838935daae4960d 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=4fe551b6be96c12334901fa1bf111a4fcedbba7db249f9566838935daae4960d 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=48, curl_handle=0x564c6f911bf0 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 run: stack=0x564c6f953ef0 is io blocked 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4df2ffd640 20 link_request req_data=0x564c6f912300 req_data->id=48, curl_handle=0x564c6f911bf0 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f968a70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f968a70:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 15 stack 0x564c6f91c0c0 end 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 run: stack=0x564c6f91c0c0 is done 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f965000:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 trim: trimmed bucket instance dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 15 stack 0x564c6f91b5f0 end 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 run: stack=0x564c6f91b5f0 is done 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 4 starting trim on bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f94ab50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f94ab50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 enqueued request req=0x564c6f9657e0 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 RGWWQ: 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 req: 0x564c6f9657e0 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e96f52900 20 run: stack=0x564c6f91b5f0 is io blocked 2026-03-20T11:46:23.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e4a7ec640 20 dequeued request req=0x564c6f9657e0 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e4a7ec640 20 RGWWQ: empty 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e4a7ec640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 : miss 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e4a7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e4a7ec640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e4a7ec640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 info.flags=0x17 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e4a7ec640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 to cache LRU end 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e4a7ec640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e4a7ec640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e4a7ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e4a7ec640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-7 : miss 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.489+0000 7f4e4a7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.488 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:23.489+0000 7fd042ad3640 -1 req 1133304097094392481 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e4a7ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e4a7ec640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-7 info.flags=0x0 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e4a7ec640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-7 to cache LRU end 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e4a7ec640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-7 : miss 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e4a7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e4a7ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e4a7ec640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-7 info.flags=0x0 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e4a7ec640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-7 to cache LRU end 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e4a7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12): adding source pipe: {s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e4a7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12): adding dest pipe: {s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e4a7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e4a7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e4a7ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f94ab50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f94ab50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f94ab50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f94ab50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f96a830:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 20 run: stack=0x564c6f91b5f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f914100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f914100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:23 +0000 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114623Z 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 3b128838447682c7dd15e1def95c772290601afd42a67818244e6cf56640db95 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114623Z 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:3b128838447682c7dd15e1def95c772290601afd42a67818244e6cf56640db95 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 7f6ff520883538f0bc2676230be3219c13101ceedd8d57c3feebb83875bff852 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=7f6ff520883538f0bc2676230be3219c13101ceedd8d57c3feebb83875bff852 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114623Z 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 20 register_request mgr=0x7ffed0a50c90 req_data->id=49, curl_handle=0x564c6f823420 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4e96f52900 20 run: stack=0x564c6f91c0c0 is io blocked 2026-03-20T11:46:23.489 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.490+0000 7f4df2ffd640 20 link_request req_data=0x564c6f918e60 req_data->id=49, curl_handle=0x564c6f823420 2026-03-20T11:46:23.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.496+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f952340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.496+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f952340:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:23.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.496+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f9113b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.496+0000 7f4e96f52900 15 stack 0x564c6f9106c0 end 2026-03-20T11:46:23.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.496+0000 7f4e96f52900 20 run: stack=0x564c6f9106c0 is done 2026-03-20T11:46:23.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.496+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.496+0000 7f4e96f52900 20 collect(): s=0x564c6f4a7b60 stack=0x564c6f9106c0 is complete 2026-03-20T11:46:23.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.496+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.496+0000 7f4e96f52900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:23.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.496+0000 7f4e96f52900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:23.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.496+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f950190:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:23.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.496+0000 7f4e96f52900 15 stack 0x564c6f4a7b60 end 2026-03-20T11:46:23.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.496+0000 7f4e96f52900 20 stack->operate() returned ret=-22 2026-03-20T11:46:23.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.496+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is done 2026-03-20T11:46:23.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.496+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.496+0000 7f4e96f52900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:23.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.496+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.497+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f909ae0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.497+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f909ae0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.497+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f909ae0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.497+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f909ae0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.497+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f969ee0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.497+0000 7f4e96f52900 15 stack 0x564c6f94bd80 end 2026-03-20T11:46:23.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.497+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is done 2026-03-20T11:46:23.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.497+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f94d7e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.497+0000 7f4e96f52900 20 collect(): s=0x564c6f4a77d0 stack=0x564c6f94bd80 is complete 2026-03-20T11:46:23.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.497+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.497+0000 7f4e96f52900 10 trimming bilogs for bucket=:dipepu-37[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42]) markers=[,,,,,,,,,,00000000001.20.6], shards=11 2026-03-20T11:46:23.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.497+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f965000:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.497+0000 7f4e96f52900 10 trimming bilog shard 10 of :dipepu-37[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42]) at marker 00000000001.20.6 2026-03-20T11:46:23.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.497+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.497+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f968a70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.497+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f968a70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.497+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42.10 2026-03-20T11:46:23.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.497+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is io blocked 2026-03-20T11:46:23.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.498+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f968a70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.498+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f968a70:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.498+0000 7f4e96f52900 15 stack 0x564c6f94bd80 end 2026-03-20T11:46:23.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.498+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.498+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is done 2026-03-20T11:46:23.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.498+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f965000:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.498+0000 7f4e96f52900 20 cr:s=0x564c6f4a77d0:op=0x564c6f94d7e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.498+0000 7f4e96f52900 20 trim: trimmed bucket instance dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:46:23.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.498+0000 7f4e96f52900 15 stack 0x564c6f4a77d0 end 2026-03-20T11:46:23.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.498+0000 7f4e96f52900 20 run: stack=0x564c6f4a77d0 is done 2026-03-20T11:46:23.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.498+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.498+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.528 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.531+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.528 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.531+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f947990:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:23.528 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.531+0000 7f4e96f52900 20 cr:s=0x564c6f953ef0:op=0x564c6f95edc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.528 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.531+0000 7f4e96f52900 15 stack 0x564c6f953ef0 end 2026-03-20T11:46:23.528 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.531+0000 7f4e96f52900 20 run: stack=0x564c6f953ef0 is done 2026-03-20T11:46:23.528 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.531+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.528 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.531+0000 7f4e96f52900 20 collect(): s=0x564c6f4a1590 stack=0x564c6f953ef0 is complete 2026-03-20T11:46:23.528 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.531+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.528 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.531+0000 7f4e96f52900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:23.528 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.531+0000 7f4e96f52900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:23.528 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.531+0000 7f4e96f52900 20 cr:s=0x564c6f4a1590:op=0x564c6f952c20:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:23.528 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.531+0000 7f4e96f52900 15 stack 0x564c6f4a1590 end 2026-03-20T11:46:23.528 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.531+0000 7f4e96f52900 20 stack->operate() returned ret=-22 2026-03-20T11:46:23.528 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.531+0000 7f4e96f52900 20 run: stack=0x564c6f4a1590 is done 2026-03-20T11:46:23.528 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.531+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.528 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.531+0000 7f4e96f52900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:23.528 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.531+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.528 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f914100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f914100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f914100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f914100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 cr:s=0x564c6f91c0c0:op=0x564c6f96a830:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 15 stack 0x564c6f91c0c0 end 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 run: stack=0x564c6f91c0c0 is done 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 collect(): s=0x564c6f91b5f0 stack=0x564c6f91c0c0 is complete 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 10 min_generation is 0 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 10 trimming bilogs for bucket=:dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) markers=[,00000000002.6.6,,00000000001.10.6,,00000000001.6.6,00000000002.6.6,00000000001.6.6,00000000001.12.6,00000000001.4.6,], shards=11 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f95a830:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 10 trimming bilog shard 1 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000002.6.6 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 10 trimming bilog shard 3 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.10.6 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 10 trimming bilog shard 5 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.6.6 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 10 trimming bilog shard 6 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000002.6.6 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 10 trimming bilog shard 7 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.6.6 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 10 trimming bilog shard 8 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.12.6 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 10 trimming bilog shard 9 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.4.6 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 run: stack=0x564c6f91b5f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f95f540:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f821700:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f919d50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 cr:s=0x564c6f9541b0:op=0x564c6f969460:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 cr:s=0x564c6f94b640:op=0x564c6f9627c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f90a760:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f9577f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f95f540:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.1 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is io blocked 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f821700:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.3 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is io blocked 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f919d50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.5 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 run: stack=0x564c6f9106c0 is io blocked 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 cr:s=0x564c6f9541b0:op=0x564c6f969460:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.6 2026-03-20T11:46:23.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 run: stack=0x564c6f9541b0 is io blocked 2026-03-20T11:46:23.530 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 cr:s=0x564c6f94b640:op=0x564c6f9627c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.530 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.7 2026-03-20T11:46:23.530 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 run: stack=0x564c6f94b640 is io blocked 2026-03-20T11:46:23.530 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f90a760:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.530 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.8 2026-03-20T11:46:23.530 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 run: stack=0x564c6f94c000 is io blocked 2026-03-20T11:46:23.530 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f9577f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.530 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.9 2026-03-20T11:46:23.530 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.532+0000 7f4e96f52900 20 run: stack=0x564c6f8210c0 is io blocked 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f821700:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 cr:s=0x564c6f4a7b60:op=0x564c6f821700:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 15 stack 0x564c6f4a7b60 end 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 run: stack=0x564c6f4a7b60 is done 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f95a830:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 run: stack=0x564c6f91b5f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 cr:s=0x564c6f94b640:op=0x564c6f9627c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 cr:s=0x564c6f94b640:op=0x564c6f9627c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 15 stack 0x564c6f94b640 end 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 run: stack=0x564c6f94b640 is done 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f95a830:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 run: stack=0x564c6f91b5f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f90a760:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 cr:s=0x564c6f94c000:op=0x564c6f90a760:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 15 stack 0x564c6f94c000 end 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 run: stack=0x564c6f94c000 is done 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f95a830:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 run: stack=0x564c6f91b5f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f919d50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 cr:s=0x564c6f9106c0:op=0x564c6f919d50:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 15 stack 0x564c6f9106c0 end 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 run: stack=0x564c6f9106c0 is done 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f95a830:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 run: stack=0x564c6f91b5f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f9577f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 cr:s=0x564c6f8210c0:op=0x564c6f9577f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 15 stack 0x564c6f8210c0 end 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 run: stack=0x564c6f8210c0 is done 2026-03-20T11:46:23.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f95a830:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 run: stack=0x564c6f91b5f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 cr:s=0x564c6f9541b0:op=0x564c6f969460:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 cr:s=0x564c6f9541b0:op=0x564c6f969460:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 15 stack 0x564c6f9541b0 end 2026-03-20T11:46:23.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 run: stack=0x564c6f9541b0 is done 2026-03-20T11:46:23.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f95a830:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 run: stack=0x564c6f91b5f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:23.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f95f540:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:23.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 cr:s=0x564c6f94bd80:op=0x564c6f95f540:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:23.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 15 stack 0x564c6f94bd80 end 2026-03-20T11:46:23.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 stack->operate() returned ret=-61 2026-03-20T11:46:23.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 run: stack=0x564c6f94bd80 is done 2026-03-20T11:46:23.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f95a830:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:23.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 cr:s=0x564c6f91b5f0:op=0x564c6f9062d0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:23.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 trim: trimmed bucket instance dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:46:23.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 15 stack 0x564c6f91b5f0 end 2026-03-20T11:46:23.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 run: stack=0x564c6f91b5f0 is done 2026-03-20T11:46:23.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:23.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f909230:27BucketTrimInstanceCollectCR: operate() returned r=-22 2026-03-20T11:46:23.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f904bc0:12BucketTrimCR: operate() 2026-03-20T11:46:23.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 writing bucket trim marker=12:69e609d8:root::dipepu-52:head 2026-03-20T11:46:23.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f914100:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:23.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f914100:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:23.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.534+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is io blocked 2026-03-20T11:46:23.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.536+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f914100:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:23.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.536+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f914100:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:23.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.536+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f914100:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:23.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.536+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f914100:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:23.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.536+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f904bc0:12BucketTrimCR: operate() 2026-03-20T11:46:23.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.536+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f952340:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:23.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.536+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f952340:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:23.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.536+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is io blocked 2026-03-20T11:46:23.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.536+0000 7f4e6b7fe640 20 trim: bucket trim completed 2026-03-20T11:46:23.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.536+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f952340:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:23.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.536+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f952340:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:23.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.536+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f952340:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:23.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.536+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f952340:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:23.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.536+0000 7f4e96f52900 20 cr:s=0x564c6f904980:op=0x564c6f904bc0:12BucketTrimCR: operate() 2026-03-20T11:46:23.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.536+0000 7f4e96f52900 4 bucket index log processing completed in 0.736417592s 2026-03-20T11:46:23.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.536+0000 7f4e96f52900 15 stack 0x564c6f904980 end 2026-03-20T11:46:23.533 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.536+0000 7f4e96f52900 20 run: stack=0x564c6f904980 is done 2026-03-20T11:46:23.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.542+0000 7f4e96f52900 20 remove_watcher() i=0 2026-03-20T11:46:23.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.542+0000 7f4e96f52900 2 removed watcher, disabling cache 2026-03-20T11:46:23.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.543+0000 7f4e96f52900 20 remove_watcher() i=2 2026-03-20T11:46:23.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.543+0000 7f4e96f52900 20 remove_watcher() i=6 2026-03-20T11:46:23.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.543+0000 7f4e96f52900 20 remove_watcher() i=5 2026-03-20T11:46:23.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.543+0000 7f4e96f52900 20 remove_watcher() i=1 2026-03-20T11:46:23.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.543+0000 7f4e96f52900 20 remove_watcher() i=4 2026-03-20T11:46:23.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.543+0000 7f4e96f52900 20 remove_watcher() i=3 2026-03-20T11:46:23.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:23.543+0000 7f4e96f52900 20 remove_watcher() i=7 2026-03-20T11:46:23.668 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'list', 'bucket.instance', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:23.668 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'list', 'bucket.instance', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:23.668 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata list bucket.instance --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:46:23.704 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:23.704 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:23.801 INFO:teuthology.orchestra.run.vm02.stdout:["dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15","dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33","dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36","dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44","dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13","dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21","dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3","dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15","dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4","dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14","dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43","dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20","dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18","dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16","dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17","dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1","dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23","dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10","dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29","dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16","dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32","dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4","dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22","dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6","dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42","dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1","dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12","dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8","dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19","dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26","dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1","dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9","dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5","dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38","dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7","dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18","dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35","dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25","dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12","dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2","dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2","dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17","dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34","dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28","dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39","dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14","dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37","dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24","dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5","dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13","dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19","dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41","dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27","dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30","dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40","dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31","dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11"] 2026-03-20T11:46:23.802 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'list', 'bucket.instance', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:23.802 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'list', 'bucket.instance', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:23.802 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata list bucket.instance --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:46:23.877 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:23.878 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:23.968 INFO:teuthology.orchestra.run.vm02.stdout:["dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15","dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33","dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36","dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44","dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13","dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21","dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3","dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15","dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4","dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14","dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43","dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20","dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18","dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16","dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17","dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1","dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23","dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10","dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29","dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16","dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32","dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4","dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22","dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6","dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42","dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1","dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12","dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8","dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19","dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26","dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1","dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9","dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5","dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38","dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7","dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18","dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35","dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25","dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12","dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2","dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2","dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17","dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34","dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28","dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39","dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14","dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37","dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10","dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24","dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5","dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13","dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19","dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41","dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27","dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30","dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40","dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31","dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11"] 2026-03-20T11:46:24.084 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'list', 'bucket.instance', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:24.084 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'list', 'bucket.instance', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:24.084 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata list bucket.instance --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:46:24.122 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:24.122 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:24.212 INFO:teuthology.orchestra.run.vm07.stdout:["dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15","dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33","dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36","dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44","dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13","dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21","dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3","dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15","dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4","dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14","dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43","dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20","dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18","dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16","dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17","dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1","dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23","dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10","dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29","dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16","dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32","dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4","dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22","dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6","dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42","dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1","dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12","dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8","dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19","dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11","dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26","dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1","dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9","dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5","dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38","dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7","dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18","dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35","dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25","dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12","dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2","dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2","dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17","dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34","testx/tenanted-bucket:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.3","dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28","dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39","dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14","dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37","dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10","dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24","dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5","dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13","dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19","dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41","dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27","dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30","dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40","dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31","dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11"] 2026-03-20T11:46:24.213 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'list', 'bucket.instance', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:24.213 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'list', 'bucket.instance', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:24.213 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata list bucket.instance --rgw-zone b2 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:46:24.289 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:24.289 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:24.373 INFO:teuthology.orchestra.run.vm07.stdout:["dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15","dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33","dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36","dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44","dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13","dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21","dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3","dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15","dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4","dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14","dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43","dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20","dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18","dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16","dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17","dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1","dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23","dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10","dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29","dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16","dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32","dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4","dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22","dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6","dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42","dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1","dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12","dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8","dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19","dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11","dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26","dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1","dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9","dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5","dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38","dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7","dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18","dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35","dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25","dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12","dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2","dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2","dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17","dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34","dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28","dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39","dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14","dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37","dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10","dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24","dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5","dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13","dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19","dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41","dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27","dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30","dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40","dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31","dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11"] 2026-03-20T11:46:24.376 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_log_trim_after_delete_bucket_primary_reshard ... ok 2026-03-20T11:46:24.509 INFO:rgw_multi.tests:create bucket zone=a1 name=dipepu-61 2026-03-20T11:46:24.554 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:24.554 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:24.554 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:46:24.590 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:24.590 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:24.669 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:46:24.681 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774007137.857782_141.1","last_update":"2026-03-20T11:45:37.857782Z"},{"marker":"1_1774007137.912792_174.1","last_update":"2026-03-20T11:45:37.912792Z"},{"marker":"1_1774007184.525561_271.1","last_update":"2026-03-20T11:46:24.525561Z"},{"marker":"1_1774007039.336409_210.1","last_update":"2026-03-20T11:43:59.336409Z"}] 2026-03-20T11:46:24.682 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:46:24.682 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:24.682 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:24.682 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:46:24.758 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:24.758 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:24.834 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:24.837+0000 7f4f35813900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:46:24.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:24.838+0000 7f4f35813900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:46:24.846 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774007137.857782_141.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:45:37.857782Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774007137.912792_174.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:37.912792Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774007184.525561_271.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:46:24.525561Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774007039.336409_210.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:59.336409Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:46:24Z"} 2026-03-20T11:46:24.846 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:46:24.846 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-61', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:46:24.846 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-61', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:46:24.846 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-61 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:46:24.927 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:24.927 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:25.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:25.014+0000 7f9476f3c900 1 waiting to reach incremental sync.. 2026-03-20T11:46:29.325 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:29.327+0000 7fcfb29b3640 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2026-03-20T11:46:29.325 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:29.327+0000 7fcfbc1c6640 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2026-03-20T11:46:29.325 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:29.327+0000 7fcfad1a8640 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2026-03-20T11:46:30.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:30.015+0000 7f9476f3c900 1 bucket sync caught up with source: 2026-03-20T11:46:30.013 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, 00000000001.53.6, , , , 00000000001.55.6, , , 00000000001.103.6, , 00000000001.69.6] 2026-03-20T11:46:30.013 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, 00000000001.53.6, , , , 00000000001.55.6, , , 00000000001.103.6, , 00000000001.69.6] 2026-03-20T11:46:30.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:30.015+0000 7f9476f3c900 0 bucket checkpoint complete 2026-03-20T11:46:30.022 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-61', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:46:30.022 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-61', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:46:30.022 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-61 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:46:30.108 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:30.108 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:30.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:30.210+0000 7f1e4192a900 1 waiting to reach incremental sync.. 2026-03-20T11:46:35.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:35.212+0000 7f1e4192a900 1 bucket sync caught up with source: 2026-03-20T11:46:35.209 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, 00000000001.75.6, , , , 00000000001.74.6, , , 00000000001.106.6, , 00000000001.81.6] 2026-03-20T11:46:35.209 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, 00000000001.75.6, , , , 00000000001.74.6, , , 00000000001.106.6, , 00000000001.81.6] 2026-03-20T11:46:35.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:35.212+0000 7f1e4192a900 0 bucket checkpoint complete 2026-03-20T11:46:35.221 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-61 zones={a2, a1} 2026-03-20T11:46:35.230 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:46:35.230 DEBUG:tasks.rgw_multi.zone_rados:o=a, v=null 2026-03-20T11:46:35.230 DEBUG:tasks.rgw_multi.zone_rados:o=b, v=null 2026-03-20T11:46:35.230 DEBUG:tasks.rgw_multi.zone_rados:o=c, v=null 2026-03-20T11:46:35.230 DEBUG:tasks.rgw_multi.zone_rados:o=d, v=null 2026-03-20T11:46:35.234 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:46:35.234 DEBUG:tasks.rgw_multi.zone_rados:o=a, v=null 2026-03-20T11:46:35.234 DEBUG:tasks.rgw_multi.zone_rados:o=b, v=null 2026-03-20T11:46:35.234 DEBUG:tasks.rgw_multi.zone_rados:o=c, v=null 2026-03-20T11:46:35.234 DEBUG:tasks.rgw_multi.zone_rados:o=d, v=null 2026-03-20T11:46:35.234 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2026-03-20T11:46:35.299 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2026-03-20T11:46:35.362 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2026-03-20T11:46:35.421 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2026-03-20T11:46:35.476 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-61 zones={a2, a1} 2026-03-20T11:46:35.476 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'reshard', '--bucket', 'dipepu-61', '--num-shards', '13', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:46:35.476 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'reshard', '--bucket', 'dipepu-61', '--num-shards', '13', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:46:35.476 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket reshard --bucket dipepu-61 --num-shards 13 --yes-i-really-mean-it --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:46:35.515 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:35.515 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:35.621 INFO:teuthology.orchestra.run.vm02.stdout:tenant: 2026-03-20T11:46:35.621 INFO:teuthology.orchestra.run.vm02.stdout:bucket name: dipepu-61 2026-03-20T11:46:35.621 INFO:teuthology.orchestra.run.vm02.stdout:start time: 2026-03-20T11:46:35.624950+0000 2026-03-20T11:46:35.624 INFO:teuthology.orchestra.run.vm02.stdout:inventory_entries: 4 2026-03-20T11:46:35.624 INFO:teuthology.orchestra.run.vm02.stdout:end time: 2026-03-20T11:46:35.628357+0000 2026-03-20T11:46:35.648 INFO:teuthology.orchestra.run.vm02.stdout:start time: 2026-03-20T11:46:35.651893+0000 2026-03-20T11:46:35.651 INFO:teuthology.orchestra.run.vm02.stdout:inc_entries: 0 2026-03-20T11:46:35.651 INFO:teuthology.orchestra.run.vm02.stdout:end time: 2026-03-20T11:46:35.654816+0000 2026-03-20T11:46:35.695 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['object', 'rm', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--bucket', 'dipepu-61', '--object', 'a', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:46:35.695 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'object', 'rm', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--bucket', 'dipepu-61', '--object', 'a', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:46:35.695 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 object rm --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --bucket dipepu-61 --object a --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:46:35.779 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:35.779 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:35.937 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['object', 'rm', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--bucket', 'dipepu-61', '--object', 'b', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:46:35.938 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'object', 'rm', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--bucket', 'dipepu-61', '--object', 'b', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:46:35.938 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 object rm --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --bucket dipepu-61 --object b --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:46:35.985 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:35.985 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:36.128 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['object', 'rm', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--bucket', 'dipepu-61', '--object', 'c', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:46:36.128 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'object', 'rm', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--bucket', 'dipepu-61', '--object', 'c', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:46:36.128 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 object rm --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --bucket dipepu-61 --object c --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:46:36.176 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:36.176 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:36.317 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['object', 'rm', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--bucket', 'dipepu-61', '--object', 'd', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:46:36.317 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'object', 'rm', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--bucket', 'dipepu-61', '--object', 'd', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:46:36.317 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 object rm --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --bucket dipepu-61 --object d --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:46:36.358 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:36.358 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:36.540 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:36.540 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:36.540 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:46:36.579 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:36.579 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:36.697 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"00000000000000000000:00000000000000009278","last_update":"2026-03-20T11:46:36.538066Z"},{"marker":"00000000000000000000:00000000000000018720","last_update":"2026-03-20T11:46:36.539433Z"},{"marker":"00000000000000000000:00000000000000013929","last_update":"2026-03-20T11:46:36.535103Z"},{"marker":"00000000000000000000:00000000000000011827","last_update":"2026-03-20T11:46:36.537012Z"}] 2026-03-20T11:46:36.697 INFO:rgw_multi.tests:starting data checkpoint for target_zone=a2 source_zone=a1 2026-03-20T11:46:36.697 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:36.697 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:36.697 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --source-zone a1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:46:36.774 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:36.774 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:36.871 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"instance_id":15702199979187085538},"markers":[{"key":0,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000008290","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:45.439897Z"}},{"key":1,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000017449","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:46:24.548972Z"}},{"key":2,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000013083","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:46:24.555380Z"}},{"key":3,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000010979","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:46:24.543391Z"}}]},"full_sync":{"total":0,"complete":0},"current_time":"2026-03-20T11:46:36Z"} 2026-03-20T11:46:36.872 WARNING:rgw_multi.tests:data of zone a2 behind zone a1: shard=0 master=00000000000000000000:00000000000000009278 target=00000000000000000000:00000000000000008290, shard=1 master=00000000000000000000:00000000000000018720 target=00000000000000000000:00000000000000017449, shard=2 master=00000000000000000000:00000000000000013929 target=00000000000000000000:00000000000000013083, shard=3 master=00000000000000000000:00000000000000011827 target=00000000000000000000:00000000000000010979 2026-03-20T11:46:38.393 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:38.395+0000 7fcfb69bb640 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2026-03-20T11:46:38.393 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:38.395+0000 7fcfad1a8640 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2026-03-20T11:46:38.394 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:38.396+0000 7fcfaf9ad640 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2026-03-20T11:46:41.872 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:41.872 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:41.872 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --source-zone a1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:46:41.909 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:41.909 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:42.001 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"instance_id":15702199979187085538},"markers":[{"key":0,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000009278","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:46:36.538066Z"}},{"key":1,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000018720","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:46:36.539433Z"}},{"key":2,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000013929","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:46:36.535103Z"}},{"key":3,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000011827","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:46:36.537012Z"}}]},"full_sync":{"total":0,"complete":0},"current_time":"2026-03-20T11:46:41Z"} 2026-03-20T11:46:42.001 INFO:rgw_multi.tests:finished data checkpoint for target_zone=a2 source_zone=a1 2026-03-20T11:46:42.001 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:42.001 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:42.001 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:46:42.081 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:42.081 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:42.176 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"00000000000000000000:00000000000000006598","last_update":"2026-03-20T11:46:36.705909Z"},{"marker":"00000000000000000000:00000000000000014908","last_update":"2026-03-20T11:46:36.700973Z"},{"marker":"00000000000000000000:00000000000000011532","last_update":"2026-03-20T11:46:36.708925Z"},{"marker":"00000000000000000000:00000000000000009143","last_update":"2026-03-20T11:46:36.704346Z"}] 2026-03-20T11:46:42.177 INFO:rgw_multi.tests:starting data checkpoint for target_zone=a1 source_zone=a2 2026-03-20T11:46:42.177 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:42.177 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--source-zone', 'a2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:42.177 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --source-zone a2 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:46:42.256 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:42.256 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:42.380 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"instance_id":5866003269306126155},"markers":[{"key":0,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000006598","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:46:36.705909Z"}},{"key":1,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000014908","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:46:36.700973Z"}},{"key":2,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000011532","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:46:36.708925Z"}},{"key":3,"val":{"status":"incremental-sync","marker":"00000000000000000000:00000000000000009143","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:46:36.704346Z"}}]},"full_sync":{"total":0,"complete":0},"current_time":"2026-03-20T11:46:42Z"} 2026-03-20T11:46:42.380 INFO:rgw_multi.tests:finished data checkpoint for target_zone=a1 source_zone=a2 2026-03-20T11:46:42.380 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '50', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:46:42.380 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '50', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:46:42.380 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bilog autotrim --rgw-sync-log-trim-max-buckets 50 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 20 --debug-ms 0 2026-03-20T11:46:42.461 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:42.461 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:42.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.482+0000 7fe839341900 20 rados->read ofs=0 len=0 2026-03-20T11:46:42.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.483+0000 7fe839341900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:46:42.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.483+0000 7fe839341900 20 rados->read ofs=0 len=0 2026-03-20T11:46:42.480 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.483+0000 7fe839341900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:46:42.480 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.483+0000 7fe839341900 20 realm test-realm 6a789b89-c2b8-45f5-82cc-fb80234d0c6c 2026-03-20T11:46:42.480 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.483+0000 7fe839341900 20 rados->read ofs=0 len=0 2026-03-20T11:46:42.480 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.483+0000 7fe839341900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:46:42.480 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.483+0000 7fe839341900 20 rados->read ofs=0 len=0 2026-03-20T11:46:42.481 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.484+0000 7fe839341900 20 rados_obj.operate() r=0 bl.length=10 2026-03-20T11:46:42.481 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.484+0000 7fe839341900 20 rados->read ofs=0 len=0 2026-03-20T11:46:42.481 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.484+0000 7fe839341900 20 rados_obj.operate() r=0 bl.length=2020 2026-03-20T11:46:42.481 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.484+0000 7fe839341900 20 rados->read ofs=0 len=0 2026-03-20T11:46:42.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.485+0000 7fe839341900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:46:42.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.485+0000 7fe839341900 20 rados->read ofs=0 len=0 2026-03-20T11:46:42.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.485+0000 7fe839341900 20 rados_obj.operate() r=0 bl.length=1030 2026-03-20T11:46:42.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.485+0000 7fe839341900 20 zone a2 found 2026-03-20T11:46:42.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.485+0000 7fe839341900 4 Realm: test-realm (6a789b89-c2b8-45f5-82cc-fb80234d0c6c) 2026-03-20T11:46:42.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.485+0000 7fe839341900 4 ZoneGroup: a (a) 2026-03-20T11:46:42.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.485+0000 7fe839341900 4 Zone: a2 (51055576-6662-409d-8abb-ecaeefd3dd80) 2026-03-20T11:46:42.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.485+0000 7fe839341900 4 using period configuration: cb2fcdce-2c1b-429a-8601-a284ee3a2381:12 2026-03-20T11:46:42.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.485+0000 7fe839341900 20 period zonegroup name a 2026-03-20T11:46:42.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.485+0000 7fe839341900 20 using current period zonegroup a 2026-03-20T11:46:42.482 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.485+0000 7fe839341900 20 rados->read ofs=0 len=0 2026-03-20T11:46:42.483 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.486+0000 7fe839341900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.483 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.486+0000 7fe839341900 20 rados->read ofs=0 len=0 2026-03-20T11:46:42.483 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.486+0000 7fe839341900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.483 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.486+0000 7fe839341900 20 init(): pipe_map (zone=4b634ebf-58f5-45bc-839a-cb4b425e773a bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:46:42.483 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.486+0000 7fe839341900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.483 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.486+0000 7fe839341900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.483 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.486+0000 7fe839341900 20 rados->read ofs=0 len=0 2026-03-20T11:46:42.483 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.486+0000 7fe839341900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.483 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.486+0000 7fe839341900 20 rados->read ofs=0 len=0 2026-03-20T11:46:42.483 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.486+0000 7fe839341900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.483 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.486+0000 7fe839341900 20 init(): pipe_map (zone=51055576-6662-409d-8abb-ecaeefd3dd80 bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:46:42.483 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.486+0000 7fe839341900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.483 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.486+0000 7fe839341900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.483 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.486+0000 7fe839341900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.483 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.486+0000 7fe839341900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.483 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.486+0000 7fe839341900 20 started sync module instance, tier type = 2026-03-20T11:46:42.483 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.486+0000 7fe839341900 20 generating connection object for zone a1 id 4b634ebf-58f5-45bc-839a-cb4b425e773a 2026-03-20T11:46:42.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.486+0000 7fe839341900 20 started zone id=51055576-6662-409d-8abb-ecaeefd3dd80 (name=a2) with tier type = 2026-03-20T11:46:42.486 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.489+0000 7fe839341900 20 add_watcher() i=3 2026-03-20T11:46:42.487 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.490+0000 7fe839341900 20 add_watcher() i=0 2026-03-20T11:46:42.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.491+0000 7fe839341900 20 add_watcher() i=6 2026-03-20T11:46:42.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.491+0000 7fe839341900 20 add_watcher() i=2 2026-03-20T11:46:42.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.491+0000 7fe839341900 20 add_watcher() i=7 2026-03-20T11:46:42.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.491+0000 7fe839341900 20 add_watcher() i=4 2026-03-20T11:46:42.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.491+0000 7fe839341900 20 add_watcher() i=5 2026-03-20T11:46:42.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.492+0000 7fe839341900 20 add_watcher() i=1 2026-03-20T11:46:42.488 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.492+0000 7fe839341900 2 all 8 watchers are set, enabling cache 2026-03-20T11:46:42.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.494+0000 7fe839341900 20 rgw_check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2026-03-20T11:46:42.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.494+0000 7fe839341900 20 rgw_check_secure_mon_conn(): mode 1 is insecure 2026-03-20T11:46:42.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.494+0000 7fe839341900 5 note: GC not initialized 2026-03-20T11:46:42.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.550+0000 7fe839341900 20 init_complete bucket index max shards: 11 2026-03-20T11:46:42.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.551+0000 7fe839341900 20 Filter name: none 2026-03-20T11:46:42.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.551+0000 7fe7e27e4640 20 reqs_thread_entry: start 2026-03-20T11:46:42.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.551+0000 7fe7e0fe1640 20 reqs_thread_entry: start 2026-03-20T11:46:42.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.553+0000 7fe839341900 10 rgw bucket trim manager: Watching bilog.trim 2026-03-20T11:46:42.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.553+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73d7050:12BucketTrimCR: operate() 2026-03-20T11:46:42.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.553+0000 7fe839341900 10 fetching active bucket counters 2026-03-20T11:46:42.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.553+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73d9b00:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:42.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.553+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73d9b00:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:42.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.553+0000 7fe839341900 20 run: stack=0x55bff73d7750 is io blocked 2026-03-20T11:46:42.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.553+0000 7fe810ff9640 20 trim: get_bucket_counters: [] 2026-03-20T11:46:42.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.553+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73d9b00:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:42.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.553+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73d9b00:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:42.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.553+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73d9b00:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:42.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.553+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73d9b00:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:42.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.553+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73d7050:12BucketTrimCR: operate() 2026-03-20T11:46:42.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.553+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:42.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.553+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:42.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.553+0000 7fe839341900 20 run: stack=0x55bff73d7750 is io blocked 2026-03-20T11:46:42.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.554+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:42.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.554+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:42.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.554+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:42.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.554+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:42.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.554+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73d7050:12BucketTrimCR: operate() 2026-03-20T11:46:42.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.554+0000 7fe839341900 10 listing cold buckets from marker=16:c59acbbb:root::dipepu-27:head 2026-03-20T11:46:42.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.554+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:14MetadataListCR: operate() 2026-03-20T11:46:42.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.554+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:14MetadataListCR: operate() 2026-03-20T11:46:42.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.554+0000 7fe839341900 20 enqueued request req=0x7fe7c8001660 2026-03-20T11:46:42.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.554+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.554+0000 7fe839341900 20 req: 0x7fe7c8001660 2026-03-20T11:46:42.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.554+0000 7fe839341900 20 run: stack=0x55bff73d7750 is io blocked 2026-03-20T11:46:42.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.554+0000 7fe7fcff9640 20 dequeued request req=0x7fe7c8001660 2026-03-20T11:46:42.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.554+0000 7fe7fcff9640 20 RGWWQ: empty 2026-03-20T11:46:42.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.554+0000 7fe7fcff9640 20 rgw async rados processor: starting metadata listing at 16:c59acbbb:root::dipepu-27:head 2026-03-20T11:46:42.552 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.556+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-27 2026-03-20T11:46:42.553 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.556+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-28 2026-03-20T11:46:42.553 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.556+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-59 2026-03-20T11:46:42.553 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.556+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:46:42.554 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.557+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:42.555 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.558+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-40 2026-03-20T11:46:42.555 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.558+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-22 2026-03-20T11:46:42.555 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.558+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:46:42.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.560+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-56 2026-03-20T11:46:42.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.560+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 2026-03-20T11:46:42.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.561+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:46:42.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.562+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:46:42.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.563+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:42.561 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.564+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-53 2026-03-20T11:46:42.561 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.564+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:46:42.562 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.565+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:46:42.563 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.566+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:46:42.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.567+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:46:42.564 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.568+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:46:42.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.568+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-34 2026-03-20T11:46:42.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.568+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-21 2026-03-20T11:46:42.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.568+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-14 2026-03-20T11:46:42.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.568+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-50 2026-03-20T11:46:42.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.568+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-57 2026-03-20T11:46:42.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.568+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:46:42.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.569+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-39 2026-03-20T11:46:42.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.569+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:42.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.569+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-37 2026-03-20T11:46:42.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.569+0000 7fe7fcff9640 20 rgw async rados processor: restarting metadata listing 2026-03-20T11:46:42.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.576+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-10 2026-03-20T11:46:42.572 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.576+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:42.578 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.581+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:42.584 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.587+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:42.592 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.595+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:42.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.602+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:42.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.609+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:42.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.618+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-19 2026-03-20T11:46:42.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.618+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-3 2026-03-20T11:46:42.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.618+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:42.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.624+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:42.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.631+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-17 2026-03-20T11:46:42.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.631+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:42.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.638+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-38 2026-03-20T11:46:42.634 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.638+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:42.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.643+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-24 2026-03-20T11:46:42.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.643+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-36 2026-03-20T11:46:42.640 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.643+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:42.646 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.649+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:42.653 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.656+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:42.659 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.662+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-44 2026-03-20T11:46:42.659 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.662+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-35 2026-03-20T11:46:42.659 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.662+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:42.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.667+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-16 2026-03-20T11:46:42.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.667+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:42.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.673+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:42.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.678+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:42.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.684+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-55 2026-03-20T11:46:42.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.684+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-13 2026-03-20T11:46:42.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.684+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-48 2026-03-20T11:46:42.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.684+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-25 2026-03-20T11:46:42.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.684+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-51 2026-03-20T11:46:42.681 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.684+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:42.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.689+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:42.690 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.693+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-46 2026-03-20T11:46:42.690 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.693+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:46:42.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.697+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:46:42.698 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.701+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:46:42.702 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.706+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-26 2026-03-20T11:46:42.702 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.706+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-20 2026-03-20T11:46:42.703 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.706+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:46:42.706 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.710+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:46:42.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.713+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-1 2026-03-20T11:46:42.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.713+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:46:42.714 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.717+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:46:42.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.721+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:46:42.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.724+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-52 2026-03-20T11:46:42.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.724+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-32 2026-03-20T11:46:42.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.724+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-42 2026-03-20T11:46:42.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.724+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:46:42.724 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.727+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:46:42.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.730+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-7 2026-03-20T11:46:42.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.730+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-54 2026-03-20T11:46:42.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.730+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-49 2026-03-20T11:46:42.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.730+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-47 2026-03-20T11:46:42.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.730+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:46:42.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.732+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:46:42.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.735+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-11 2026-03-20T11:46:42.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.735+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-12 2026-03-20T11:46:42.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.735+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-41 2026-03-20T11:46:42.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.735+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-23 2026-03-20T11:46:42.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.735+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-15 2026-03-20T11:46:42.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.735+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-45 2026-03-20T11:46:42.732 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.735+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:46:42.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.738+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:46:42.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.741+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-9 2026-03-20T11:46:42.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.741+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:46:42.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.743+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-29 2026-03-20T11:46:42.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.743+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-31 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7fcff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:14MetadataListCR: operate() 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:14MetadataListCR: operate() 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:14MetadataListCR: operate() 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:14MetadataListCR: operate() 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73d7050:12BucketTrimCR: operate() 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 4 collected 50 buckets for trim 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73d8af0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 4 starting trim on bucket=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 4 starting trim on bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 cr:s=0x55bff72e71d0:op=0x55bff73dda70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 4 starting trim on bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 cr:s=0x55bff72fa2e0:op=0x55bff73ded40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 4 starting trim on bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73e0010:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e0760:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 cr:s=0x55bff72e71d0:op=0x55bff73e0eb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 cr:s=0x55bff72fa2e0:op=0x55bff73e1600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73e0010:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 enqueued request req=0x55bff73dc110 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 req: 0x55bff73dc110 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 run: stack=0x55bff73dbd10 is io blocked 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e0760:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 enqueued request req=0x55bff73e1d50 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 req: 0x55bff73dc110 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 req: 0x55bff73e1d50 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is io blocked 2026-03-20T11:46:42.743 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 cr:s=0x55bff72e71d0:op=0x55bff73e0eb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 enqueued request req=0x55bff73e2170 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 req: 0x55bff73dc110 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 req: 0x55bff73e1d50 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 req: 0x55bff73e2170 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 run: stack=0x55bff72e71d0 is io blocked 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 cr:s=0x55bff72fa2e0:op=0x55bff73e1600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 enqueued request req=0x55bff73e2590 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 req: 0x55bff73dc110 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 req: 0x55bff73e1d50 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 req: 0x55bff73e2170 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 req: 0x55bff73e2590 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe839341900 20 run: stack=0x55bff72fa2e0 is io blocked 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f77fe640 20 dequeued request req=0x55bff73dc110 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f77fe640 20 RGWWQ: 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f77fe640 20 req: 0x55bff73e1d50 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f77fe640 20 req: 0x55bff73e2170 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f77fe640 20 req: 0x55bff73e2590 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f77fe640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 : miss 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f77fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f7fff640 20 dequeued request req=0x55bff73e1d50 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f7fff640 20 RGWWQ: 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f7fff640 20 req: 0x55bff73e2170 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f7fff640 20 req: 0x55bff73e2590 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f67fc640 20 dequeued request req=0x55bff73e2170 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f67fc640 20 RGWWQ: 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f67fc640 20 req: 0x55bff73e2590 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f67fc640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 : miss 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f67fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f6ffd640 20 dequeued request req=0x55bff73e2590 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f6ffd640 20 RGWWQ: empty 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f6ffd640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 : miss 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f6ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f7fff640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 : miss 2026-03-20T11:46:42.744 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f7fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f67fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:42.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f67fc640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 info.flags=0x17 2026-03-20T11:46:42.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f67fc640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 to cache LRU end 2026-03-20T11:46:42.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f67fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:42.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f67fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:42.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f67fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:42.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f67fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-54 : miss 2026-03-20T11:46:42.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f67fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f6ffd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:42.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f6ffd640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 info.flags=0x17 2026-03-20T11:46:42.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f6ffd640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 to cache LRU end 2026-03-20T11:46:42.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f6ffd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:42.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f6ffd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:42.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f6ffd640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:46:42.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f6ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:46:42.746 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f6ffd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-32 : miss 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.746+0000 7fe7f6ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f77fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=387 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f77fe640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 info.flags=0x17 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f77fe640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 to cache LRU end 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f77fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f77fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f77fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f77fe640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-61 : miss 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f77fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f7fff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f7fff640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 info.flags=0x17 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f7fff640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 to cache LRU end 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f7fff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f7fff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f7fff640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f77fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f77fe640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-61 info.flags=0x0 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f77fe640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-61 to cache LRU end 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f77fe640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-61 : miss 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f77fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f7fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f7fff640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-34 : miss 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f7fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f67fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f67fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-54 info.flags=0x0 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f67fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-54 to cache LRU end 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f67fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-54 : miss 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f67fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f6ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f6ffd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-32 info.flags=0x0 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f6ffd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-32 to cache LRU end 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f6ffd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-32 : miss 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f6ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f77fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f77fe640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-61 info.flags=0x0 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f77fe640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-61 to cache LRU end 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f77fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21): adding source pipe: {s={b=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f77fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21): adding dest pipe: {s={b=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f77fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f77fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f77fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f7fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f7fff640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-34 info.flags=0x0 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f7fff640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-34 to cache LRU end 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f7fff640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-34 : miss 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f7fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73e0010:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73e0010:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73e0010:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73e0010:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73d8af0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe839341900 20 cr:s=0x55bff73da0b0:op=0x55bff73e29b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73d8af0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe839341900 20 run: stack=0x55bff73dbd10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe839341900 20 cr:s=0x55bff73da0b0:op=0x55bff73e0010:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe839341900 20 cr:s=0x55bff73da0b0:op=0x55bff73e0010:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f67fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f67fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-54 info.flags=0x0 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f67fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-54 to cache LRU end 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f67fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14): adding source pipe: {s={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f67fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14): adding dest pipe: {s={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f67fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f67fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe7f67fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.747+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 1a4980e524fb57d7ec99756868498f4f5bda8f747e0912ac0251ae2a16ecb9d0 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114642Z 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:1a4980e524fb57d7ec99756868498f4f5bda8f747e0912ac0251ae2a16ecb9d0 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 66a55e61fd28a9a558fd2ed4a5b4cb90a6fbdb416b5e50a364ead115ca34c262 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=66a55e61fd28a9a558fd2ed4a5b4cb90a6fbdb416b5e50a364ead115ca34c262 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114642Z 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=0, curl_handle=0x55bff73e4760 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 run: stack=0x55bff73da0b0 is io blocked 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 cr:s=0x55bff72e71d0:op=0x55bff73e0eb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 cr:s=0x55bff72e71d0:op=0x55bff73e0eb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 cr:s=0x55bff72e71d0:op=0x55bff73e0eb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 cr:s=0x55bff72e71d0:op=0x55bff73e0eb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 cr:s=0x55bff72e71d0:op=0x55bff73dda70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff73e6ce0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 cr:s=0x55bff72e71d0:op=0x55bff73dda70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 run: stack=0x55bff72e71d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff73e7330:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff73e7330:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 2634c57c1aab81bb02ea431e7a97019a38f99aa3aed1d58a8d03a65bd34e9052 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114642Z 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2634c57c1aab81bb02ea431e7a97019a38f99aa3aed1d58a8d03a65bd34e9052 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 4abeb221bd908341984ba26e51ed5dd09875d19510831c9e90f2d1d3f4c271f3 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=4abeb221bd908341984ba26e51ed5dd09875d19510831c9e90f2d1d3f4c271f3 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114642Z 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=1, curl_handle=0x55bff73e8f10 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 run: stack=0x55bff73e14a0 is io blocked 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73e5370 req_data->id=0, curl_handle=0x55bff73e4760 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73e9620 req_data->id=1, curl_handle=0x55bff73e8f10 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe7f7fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe7f7fff640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-34 info.flags=0x0 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe7f7fff640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-34 to cache LRU end 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe7f7fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39): adding source pipe: {s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe7f7fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39): adding dest pipe: {s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe7f7fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe7f7fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe7f7fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e0760:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e0760:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e0760:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e0760:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff73eadc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff73eb3e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff73eb3e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = ca2b23d2defddd869572e97810e331ec404df82db571a8ce5618263990a3fecd 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114642Z 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:ca2b23d2defddd869572e97810e331ec404df82db571a8ce5618263990a3fecd 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = ca45e635283402332b7d1dc15a0033a6b8e4b78401b618028b8b6d17a0b713a8 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=ca45e635283402332b7d1dc15a0033a6b8e4b78401b618028b8b6d17a0b713a8 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114642Z 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=2, curl_handle=0x55bff73ecfc0 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is io blocked 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe7f6ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe7f6ffd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-32 info.flags=0x0 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe7f6ffd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-32 to cache LRU end 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe7f6ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37): adding source pipe: {s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe7f6ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37): adding dest pipe: {s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe7f6ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe7f6ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe7f6ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73ed6d0 req_data->id=2, curl_handle=0x55bff73ecfc0 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 cr:s=0x55bff72fa2e0:op=0x55bff73e1600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 cr:s=0x55bff72fa2e0:op=0x55bff73e1600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 cr:s=0x55bff72fa2e0:op=0x55bff73e1600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 cr:s=0x55bff72fa2e0:op=0x55bff73e1600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 cr:s=0x55bff72fa2e0:op=0x55bff73ded40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff73eee70:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 cr:s=0x55bff72fa2e0:op=0x55bff73ded40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 run: stack=0x55bff72fa2e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff73ef620:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff73ef620:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.750 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 9ce8f396ceeb5dfaffde779b78d7ed19d004a5dd861152c891aa4707d7f95313 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114642Z 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr:9ce8f396ceeb5dfaffde779b78d7ed19d004a5dd861152c891aa4707d7f95313 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 4ebd0ccf65c78825d53fb5bf5bbec835e0483719cee31df07943888662022f43 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=4ebd0ccf65c78825d53fb5bf5bbec835e0483719cee31df07943888662022f43 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114642Z 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=3, curl_handle=0x55bff73f1200 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe839341900 20 run: stack=0x55bff73ef490 is io blocked 2026-03-20T11:46:42.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.748+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73f1910 req_data->id=3, curl_handle=0x55bff73f1200 2026-03-20T11:46:42.751 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:42.751+0000 7f12f02d0640 -1 req 7418610218630146232 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff73eb3e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff73eb3e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff73eadc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 15 stack 0x55bff73e0d20 end 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is done 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff73ef620:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff73ef620:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff73e7330:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff73da0b0:op=0x55bff73e0010:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 collect(): s=0x55bff72f1ec0 stack=0x55bff73e0d20 is complete 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 15 stack 0x55bff72f1ec0 end 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 stack->operate() returned ret=-22 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is done 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff73eee70:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 15 stack 0x55bff73ef490 end 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 run: stack=0x55bff73ef490 is done 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff73e7330:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff73da0b0:op=0x55bff73e0010:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff72fa2e0:op=0x55bff73ded40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 collect(): s=0x55bff72fa2e0 stack=0x55bff73ef490 is complete 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff72fa2e0:op=0x55bff73ded40:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 15 stack 0x55bff72fa2e0 end 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 stack->operate() returned ret=-22 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 run: stack=0x55bff72fa2e0 is done 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff73e7330:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff73da0b0:op=0x55bff73e0010:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 4 starting trim on bucket=dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:42.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff73e7330:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff73da0b0:op=0x55bff73e0010:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e7960:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff72fa2e0:op=0x55bff73ded40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 4 starting trim on bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff73e6ce0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 15 stack 0x55bff73e14a0 end 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 run: stack=0x55bff73e14a0 is done 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff73da0b0:op=0x55bff73e29b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 15 stack 0x55bff73da0b0 end 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 run: stack=0x55bff73da0b0 is done 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e7960:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 enqueued request req=0x55bff73db8f0 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 req: 0x55bff73db8f0 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is io blocked 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff72fa2e0:op=0x55bff73e2fd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff72e71d0:op=0x55bff73dda70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 collect(): s=0x55bff72e71d0 stack=0x55bff73e14a0 is complete 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 10 trimming bilogs for bucket=:dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) markers=[00000000004.65.1,00000000002.57.1,00000000004.95.1,00000000002.36.1,00000000004.41.1,00000000002.67.1,00000000002.53.1,00000000004.82.1,00000000002.67.1,00000000002.123.1,00000000002.39.1], shards=11 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73d8af0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 collect(): s=0x55bff73dbd10 stack=0x55bff73da0b0 is complete 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff72fa2e0:op=0x55bff73e2fd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 enqueued request req=0x55bff73e2590 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 req: 0x55bff73db8f0 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 req: 0x55bff73e2590 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 run: stack=0x55bff72fa2e0 is io blocked 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 20 cr:s=0x55bff72e71d0:op=0x55bff742c300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 10 trimming bilog shard 0 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.65.1 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 10 trimming bilog shard 1 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.57.1 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 10 trimming bilog shard 2 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.95.1 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 10 trimming bilog shard 3 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.36.1 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 10 trimming bilog shard 4 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.41.1 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 10 trimming bilog shard 5 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.67.1 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 10 trimming bilog shard 6 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.53.1 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 10 trimming bilog shard 7 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.82.1 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 10 trimming bilog shard 8 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.67.1 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 10 trimming bilog shard 9 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.123.1 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.752+0000 7fe839341900 10 trimming bilog shard 10 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.39.1 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 run: stack=0x55bff72e71d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73e6ce0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.0 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.1 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.2 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.3 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.4 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.5 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe7f5ffb640 20 dequeued request req=0x55bff73db8f0 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe7f5ffb640 20 RGWWQ: 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe7f5ffb640 20 req: 0x55bff73e2590 2026-03-20T11:46:42.753 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.6 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.7 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.8 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.9 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.10 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 run: stack=0x55bff73dbd10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff73f7af0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff7426af0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff7429b00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff7426880:op=0x55bff73f3490:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff73f0960:op=0x55bff7428cc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7427e20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff74230d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff742bef0:op=0x55bff742e430:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff73f3050:op=0x55bff742ec30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe7f5ffb640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 : miss 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff73f4260:op=0x55bff7424200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff73e44d0:op=0x55bff7424a00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff73ec000:op=0x55bff73e0010:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff742b2d0:op=0x55bff73e7330:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff73f0340:op=0x55bff73eee50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff742f430:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff73e4030:op=0x55bff73e2910:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff73f1540:op=0x55bff73e8f30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff7426650:op=0x55bff742abb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff74238d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff7427a60:op=0x55bff7425200:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff73eb300:op=0x55bff742c910:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe7f5ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff742a510:op=0x55bff742cf00:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff73f7af0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.0 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe7f57fa640 20 dequeued request req=0x55bff73e2590 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe7f57fa640 20 RGWWQ: empty 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe7f57fa640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 : miss 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 run: stack=0x55bff73e14a0 is io blocked 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff7426af0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe7f57fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.1 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 run: stack=0x55bff73ef490 is io blocked 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff7429b00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.2 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is io blocked 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff7426880:op=0x55bff73f3490:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.3 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 run: stack=0x55bff7426880 is io blocked 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff73f0960:op=0x55bff7428cc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.4 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 run: stack=0x55bff73f0960 is io blocked 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7427e20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.5 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 run: stack=0x55bff742c160 is io blocked 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff74230d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.6 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is io blocked 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff742bef0:op=0x55bff742e430:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.7 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 run: stack=0x55bff742bef0 is io blocked 2026-03-20T11:46:42.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 cr:s=0x55bff73f3050:op=0x55bff742ec30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.8 2026-03-20T11:46:42.755 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:42.752+0000 7f120f10e640 -1 req 4621229037991075678 0.003000004s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:42.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe7f5ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=386 2026-03-20T11:46:42.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe7f5ffb640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 info.flags=0x17 2026-03-20T11:46:42.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe7f5ffb640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 to cache LRU end 2026-03-20T11:46:42.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe7f5ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:42.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe7f5ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:42.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe7f5ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 2026-03-20T11:46:42.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe7f5ffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-5 : miss 2026-03-20T11:46:42.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe7f5ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe7f57fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:42.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe7f57fa640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 info.flags=0x17 2026-03-20T11:46:42.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe7f57fa640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 to cache LRU end 2026-03-20T11:46:42.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe7f57fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:42.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe7f57fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:42.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.753+0000 7fe7f57fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:46:42.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe7f57fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-19 : miss 2026-03-20T11:46:42.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe7f57fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe7f5ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe7f5ffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-5 info.flags=0x0 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe7f5ffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-5 to cache LRU end 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe7f5ffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-5 : miss 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe7f5ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe839341900 20 run: stack=0x55bff73f3050 is io blocked 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe839341900 20 cr:s=0x55bff73f4260:op=0x55bff7424200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.9 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe839341900 20 run: stack=0x55bff73f4260 is io blocked 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe839341900 20 cr:s=0x55bff73e44d0:op=0x55bff7424a00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.10 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe839341900 20 run: stack=0x55bff73e44d0 is io blocked 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe839341900 20 cr:s=0x55bff73ec000:op=0x55bff73e0010:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe839341900 20 run: stack=0x55bff73ec000 is io blocked 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe839341900 20 cr:s=0x55bff742b2d0:op=0x55bff73e7330:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe839341900 20 run: stack=0x55bff742b2d0 is io blocked 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe839341900 20 cr:s=0x55bff73f0340:op=0x55bff73eee50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe839341900 20 run: stack=0x55bff73f0340 is io blocked 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff742f430:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe839341900 20 run: stack=0x55bff73ec910 is io blocked 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe839341900 20 cr:s=0x55bff73e4030:op=0x55bff73e2910:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe839341900 20 run: stack=0x55bff73e4030 is io blocked 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe839341900 20 cr:s=0x55bff73f1540:op=0x55bff73e8f30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe839341900 20 run: stack=0x55bff73f1540 is io blocked 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe839341900 20 cr:s=0x55bff7426650:op=0x55bff742abb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe839341900 20 run: stack=0x55bff7426650 is io blocked 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff74238d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe839341900 20 run: stack=0x55bff74261a0 is io blocked 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe839341900 20 cr:s=0x55bff7427a60:op=0x55bff7425200:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe839341900 20 run: stack=0x55bff7427a60 is io blocked 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe839341900 20 cr:s=0x55bff73eb300:op=0x55bff742c910:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe839341900 20 run: stack=0x55bff73eb300 is io blocked 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.754+0000 7fe839341900 20 cr:s=0x55bff742a510:op=0x55bff742cf00:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe839341900 20 run: stack=0x55bff742a510 is io blocked 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe839341900 20 cr:s=0x55bff7426880:op=0x55bff73f3490:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe839341900 20 cr:s=0x55bff7426880:op=0x55bff73f3490:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe839341900 15 stack 0x55bff7426880 end 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe839341900 20 run: stack=0x55bff7426880 is done 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff7429b00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff7429b00:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe839341900 15 stack 0x55bff73e0d20 end 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is done 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff73f7af0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff73f7af0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe839341900 15 stack 0x55bff73e14a0 end 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe839341900 20 run: stack=0x55bff73e14a0 is done 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe839341900 20 cr:s=0x55bff72e71d0:op=0x55bff742c300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe839341900 20 cr:s=0x55bff72e71d0:op=0x55bff742c300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe839341900 20 cr:s=0x55bff72e71d0:op=0x55bff742c300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe839341900 20 run: stack=0x55bff72e71d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe7f57fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe7f57fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-19 info.flags=0x0 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe7f57fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-19 to cache LRU end 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe7f57fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-19 : miss 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe7f57fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe7f5ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe7f5ffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-5 info.flags=0x0 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe7f5ffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-5 to cache LRU end 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.756+0000 7fe7f5ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10): adding source pipe: {s={b=dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe7f5ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10): adding dest pipe: {s={b=dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe7f5ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe7f5ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe7f5ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff73f0960:op=0x55bff7428cc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff73f0960:op=0x55bff7428cc0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 15 stack 0x55bff73f0960 end 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 run: stack=0x55bff73f0960 is done 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff72e71d0:op=0x55bff742c300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 run: stack=0x55bff72e71d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff7426af0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff7426af0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 15 stack 0x55bff73ef490 end 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 run: stack=0x55bff73ef490 is done 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e7960:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff73e44d0:op=0x55bff7424a00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff73e44d0:op=0x55bff7424a00:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 15 stack 0x55bff73e44d0 end 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 run: stack=0x55bff73e44d0 is done 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff73ec000:op=0x55bff73e0010:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff7427a60:op=0x55bff7425200:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff73e4030:op=0x55bff73e2910:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff742bef0:op=0x55bff742e430:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff742bef0:op=0x55bff742e430:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 15 stack 0x55bff742bef0 end 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 run: stack=0x55bff742bef0 is done 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7427e20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7427e20:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 15 stack 0x55bff742c160 end 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 run: stack=0x55bff742c160 is done 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff73f3050:op=0x55bff742ec30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff73f3050:op=0x55bff742ec30:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 15 stack 0x55bff73f3050 end 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 run: stack=0x55bff73f3050 is done 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff74230d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff74230d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 15 stack 0x55bff73f32f0 end 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is done 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff72e71d0:op=0x55bff742c300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e7960:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff73ec000:op=0x55bff73e0010:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff7427a60:op=0x55bff7425200:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff73e4030:op=0x55bff73e2910:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff72e71d0:op=0x55bff742c300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e7960:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff73ec000:op=0x55bff73e0010:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff7427a60:op=0x55bff7425200:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff73e4030:op=0x55bff73e2910:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff72e71d0:op=0x55bff742c300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e7960:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff73ec000:op=0x55bff73e0010:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 15 stack 0x55bff73ec000 end 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 run: stack=0x55bff73ec000 is done 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff7427a60:op=0x55bff7425200:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 15 stack 0x55bff7427a60 end 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 run: stack=0x55bff7427a60 is done 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff73e4030:op=0x55bff73e2910:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 15 stack 0x55bff73e4030 end 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 run: stack=0x55bff73e4030 is done 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff72e71d0:op=0x55bff742c300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73e6ce0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff72e71d0:op=0x55bff742c300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff73e7920:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe7f57fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe7f57fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-19 info.flags=0x0 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe7f57fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-19 to cache LRU end 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe7f57fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24): adding source pipe: {s={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe7f57fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24): adding dest pipe: {s={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe7f57fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73e6ce0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe7f57fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff72e71d0:op=0x55bff742c300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 run: stack=0x55bff72e71d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff73f7af0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff73f0340:op=0x55bff73eee50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff74238d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff7426650:op=0x55bff742abb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73e6ce0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 run: stack=0x55bff73dbd10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff73f7af0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe7f57fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.759 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 866614ec4c7737db691138f47355e9d8916f4dedd2642b56e0d06092379ccbf9 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.757+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114642Z 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:866614ec4c7737db691138f47355e9d8916f4dedd2642b56e0d06092379ccbf9 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = bdcc6ae0cfb9e7f12d4cd6dd461cfdb896e5ce568e967e073a5977a2ddad2ac1 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=bdcc6ae0cfb9e7f12d4cd6dd461cfdb896e5ce568e967e073a5977a2ddad2ac1 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114642Z 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=4, curl_handle=0x55bff73e4760 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is io blocked 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff73f0340:op=0x55bff73eee50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff74238d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff7426650:op=0x55bff742abb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff72fa2e0:op=0x55bff73e2fd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff73f0340:op=0x55bff73eee50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff74238d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff7426650:op=0x55bff742abb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff72fa2e0:op=0x55bff73e2fd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff73f0340:op=0x55bff73eee50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 15 stack 0x55bff73f0340 end 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 run: stack=0x55bff73f0340 is done 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff74238d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 15 stack 0x55bff74261a0 end 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 run: stack=0x55bff74261a0 is done 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff7426650:op=0x55bff742abb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 15 stack 0x55bff7426650 end 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73e5370 req_data->id=4, curl_handle=0x55bff73e4760 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 run: stack=0x55bff7426650 is done 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff72fa2e0:op=0x55bff73e2fd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73e6ce0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff72fa2e0:op=0x55bff73e2fd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73e6ce0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff72fa2e0:op=0x55bff73ded40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73e6ce0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 run: stack=0x55bff73dbd10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff73ec000:op=0x55bff7426af0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff72fa2e0:op=0x55bff73ded40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 run: stack=0x55bff72fa2e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff73ec000:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff73ec000:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.760 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 9d36b50daac0fb6366921477bac5581485b2046a911c177931a1b14fd620e007 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114642Z 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:9d36b50daac0fb6366921477bac5581485b2046a911c177931a1b14fd620e007 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 677b99b3b69cdba947e4bd4aa5558eb28c865938d1543f9d2a2345840e0f09b7 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=677b99b3b69cdba947e4bd4aa5558eb28c865938d1543f9d2a2345840e0f09b7 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114642Z 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=5, curl_handle=0x55bff73e8f10 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 run: stack=0x55bff73ec000 is io blocked 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff742a510:op=0x55bff742cf00:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff742a510:op=0x55bff742cf00:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff742a510:op=0x55bff742cf00:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff742a510:op=0x55bff742cf00:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 15 stack 0x55bff742a510 end 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 run: stack=0x55bff742a510 is done 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73e6ce0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 run: stack=0x55bff73dbd10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff73f4260:op=0x55bff7424200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff73f4260:op=0x55bff7424200:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 15 stack 0x55bff73f4260 end 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 run: stack=0x55bff73f4260 is done 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff72e71d0:op=0x55bff742c300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff72e71d0:op=0x55bff73dda70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 trim: trimmed bucket instance dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 15 stack 0x55bff72e71d0 end 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 run: stack=0x55bff72e71d0 is done 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73dda70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 4 starting trim on bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73e2f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73e2f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 enqueued request req=0x55bff73e1d50 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 req: 0x55bff73e1d50 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 run: stack=0x55bff74261a0 is io blocked 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff742f430:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff742f430:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff742f430:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff742f430:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 15 stack 0x55bff73ec910 end 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 run: stack=0x55bff73ec910 is done 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73e6ce0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe839341900 20 run: stack=0x55bff73dbd10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe7f4ff9640 20 dequeued request req=0x55bff73e1d50 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe7f4ff9640 20 RGWWQ: empty 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe7f4ff9640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 : miss 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.758+0000 7fe7f4ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73e9620 req_data->id=5, curl_handle=0x55bff73e8f10 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 20 cr:s=0x55bff742b2d0:op=0x55bff73e7330:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 20 cr:s=0x55bff742b2d0:op=0x55bff73e7330:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 20 cr:s=0x55bff73eb300:op=0x55bff742c910:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 20 cr:s=0x55bff73f1540:op=0x55bff73e8f30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 20 cr:s=0x55bff742b2d0:op=0x55bff73e7330:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 20 cr:s=0x55bff73eb300:op=0x55bff742c910:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 20 cr:s=0x55bff73f1540:op=0x55bff73e8f30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 20 cr:s=0x55bff742b2d0:op=0x55bff73e7330:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 15 stack 0x55bff742b2d0 end 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 20 run: stack=0x55bff742b2d0 is done 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 20 cr:s=0x55bff73eb300:op=0x55bff742c910:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 20 cr:s=0x55bff73f1540:op=0x55bff73e8f30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73e6ce0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 20 run: stack=0x55bff73dbd10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 20 cr:s=0x55bff73eb300:op=0x55bff742c910:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 15 stack 0x55bff73eb300 end 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 20 run: stack=0x55bff73eb300 is done 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 20 cr:s=0x55bff73f1540:op=0x55bff73e8f30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 15 stack 0x55bff73f1540 end 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 20 run: stack=0x55bff73f1540 is done 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73e6ce0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73e6ce0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73d8af0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73e7330:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73e7330:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 20 enqueued request req=0x55bff73eba50 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 20 req: 0x55bff73eba50 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe7f4ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe7f4ff9640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 info.flags=0x17 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe7f4ff9640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 to cache LRU end 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe7f4ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe7f4ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe7f4ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe839341900 20 run: stack=0x55bff73dbd10 is io blocked 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe7f4ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-4 : miss 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe7f4ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe7effff640 20 dequeued request req=0x55bff73eba50 2026-03-20T11:46:42.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.759+0000 7fe7effff640 20 RGWWQ: empty 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe7f4ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe7f4ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-4 info.flags=0x0 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe7f4ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-4 to cache LRU end 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe7f4ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-4 : miss 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe7f4ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe7f4ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe7f4ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-4 info.flags=0x0 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe7f4ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-4 to cache LRU end 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe7f4ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5): adding source pipe: {s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe7f4ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5): adding dest pipe: {s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe7f4ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe7f4ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe7f4ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73e2f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73e2f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73e2f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73e2f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73dda70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f5c9b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73dda70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 20 run: stack=0x55bff74261a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73e2f80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73e2f80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:42.763 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 2f2aef3a3902f117a4b8c39d2276b6bb2d85f40e653aca639d8e514ee751ea65 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114642Z 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2f2aef3a3902f117a4b8c39d2276b6bb2d85f40e653aca639d8e514ee751ea65 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 246147abcf05e545711c18faa23c78a8ce08eddebc75d37765015338e707ffcc 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=246147abcf05e545711c18faa23c78a8ce08eddebc75d37765015338e707ffcc 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114642Z 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=6, curl_handle=0x55bff73f1200 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.760+0000 7fe839341900 20 run: stack=0x55bff73ec910 is io blocked 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.761+0000 7fe7effff640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 info.flags=0x11 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.761+0000 7fe7effff640 10 rgw async rados processor: moving a2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 to cache LRU end 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.761+0000 7fe7effff640 10 rgw async rados processor: distributing notification oid=a2.rgw.control:notify.6 cni=[op: 0, obj: a2.rgw.meta:root:.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21, ofs0, ns] 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.761+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73f1910 req_data->id=6, curl_handle=0x55bff73f1200 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.762+0000 7fe810ff9640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 343597383943 cookie 94282975411920 notifier 6505 bl.length()=567 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.762+0000 7fe810ff9640 10 rgw watcher librados: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 info.flags=0x11 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.762+0000 7fe810ff9640 10 rgw watcher librados: moving a2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 to cache LRU end 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.762+0000 7fe7effff640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-61[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21]): orig_sources= new_sources= 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.762+0000 7fe7effff640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-61[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21]): potential sources added=[] removed=[] 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.762+0000 7fe7effff640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-61[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21]): orig_dests= new_dests= 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.762+0000 7fe7effff640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-61[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21]): potential dests added=[] removed=[] 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.762+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73e7330:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.762+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73e7330:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.762+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73e7330:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.762+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73e7330:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.762+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73d8af0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.762+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff7457310:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.762+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff7457310:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.762+0000 7fe839341900 20 enqueued request req=0x55bff6efaf40 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.762+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.762+0000 7fe839341900 20 req: 0x55bff6efaf40 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.762+0000 7fe839341900 20 run: stack=0x55bff73dbd10 is io blocked 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.762+0000 7fe7ef7fe640 20 dequeued request req=0x55bff6efaf40 2026-03-20T11:46:42.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.762+0000 7fe7ef7fe640 20 RGWWQ: empty 2026-03-20T11:46:42.764 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:42.763+0000 7f123715e640 -1 req 9939493251216968517 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:42.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.763+0000 7fe7ef7fe640 10 rgw async rados processor: removing a2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 from cache 2026-03-20T11:46:42.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.763+0000 7fe7ef7fe640 10 rgw async rados processor: distributing notification oid=a2.rgw.control:notify.6 cni=[op: 1, obj: a2.rgw.meta:root:.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21, ofs0, ns] 2026-03-20T11:46:42.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe810ff9640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 343597383944 cookie 94282975411920 notifier 6505 bl.length()=206 2026-03-20T11:46:42.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff7457310:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:42.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff7457310:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:42.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff7457310:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:42.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff7457310:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:42.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73d8af0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe839341900 20 trim: trimmed bucket instance dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:46:42.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe839341900 15 stack 0x55bff73dbd10 end 2026-03-20T11:46:42.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe839341900 20 run: stack=0x55bff73dbd10 is done 2026-03-20T11:46:42.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:42.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73d8af0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe839341900 4 starting trim on bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:42.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe839341900 20 enqueued request req=0x55bff73ec720 2026-03-20T11:46:42.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe839341900 20 req: 0x55bff73ec720 2026-03-20T11:46:42.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe839341900 20 run: stack=0x55bff73dbd10 is io blocked 2026-03-20T11:46:42.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe7eeffd640 20 dequeued request req=0x55bff73ec720 2026-03-20T11:46:42.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe7eeffd640 20 RGWWQ: empty 2026-03-20T11:46:42.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe7eeffd640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 : miss 2026-03-20T11:46:42.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe7eeffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe7eeffd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe7eeffd640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 info.flags=0x17 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe7eeffd640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 to cache LRU end 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe7eeffd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe7eeffd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe7eeffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe7eeffd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-53 : miss 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.764+0000 7fe7eeffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe7eeffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe7eeffd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-53 info.flags=0x0 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe7eeffd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-53 to cache LRU end 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe7eeffd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-53 : miss 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe7eeffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe7eeffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe7eeffd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-53 info.flags=0x0 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe7eeffd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-53 to cache LRU end 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe7eeffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13): adding source pipe: {s={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe7eeffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13): adding dest pipe: {s={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe7eeffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe7eeffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe7eeffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73d8af0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7457310:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73d8af0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 20 run: stack=0x55bff73dbd10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7428cc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7428cc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.766 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.767 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:42.767 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 7eb41f24a309a7a5055820156c3243303ac4704e9361ec7d2923d4b208536c5e 2026-03-20T11:46:42.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:42.767 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114642Z 2026-03-20T11:46:42.767 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:42.767 INFO:teuthology.orchestra.run.vm02.stderr:7eb41f24a309a7a5055820156c3243303ac4704e9361ec7d2923d4b208536c5e 2026-03-20T11:46:42.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:42.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:42.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:42.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:42.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 7346685269f7176ec1690a881f0ac007b215830b5981c23d401b614336571318 2026-03-20T11:46:42.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=7346685269f7176ec1690a881f0ac007b215830b5981c23d401b614336571318 2026-03-20T11:46:42.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114642Z 2026-03-20T11:46:42.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:42.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=7, curl_handle=0x55bff73ecfc0 2026-03-20T11:46:42.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe7e0fe1640 20 link_request req_data=0x7fe7d80062a0 req_data->id=7, curl_handle=0x55bff73ecfc0 2026-03-20T11:46:42.767 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.765+0000 7fe839341900 20 run: stack=0x55bff742c160 is io blocked 2026-03-20T11:46:42.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff73f7af0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff73f7af0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:42.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff73e7920:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 15 stack 0x55bff73f32f0 end 2026-03-20T11:46:42.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is done 2026-03-20T11:46:42.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 collect(): s=0x55bff72f1ec0 stack=0x55bff73f32f0 is complete 2026-03-20T11:46:42.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:42.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e8f30:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:42.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.0 2026-03-20T11:46:42.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.1 2026-03-20T11:46:42.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.2 2026-03-20T11:46:42.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.3 2026-03-20T11:46:42.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.4 2026-03-20T11:46:42.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.5 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.6 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.7 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.8 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.9 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.10 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff742abb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff73e2910:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 cr:s=0x55bff73f0960:op=0x55bff73e0010:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 cr:s=0x55bff7426880:op=0x55bff73e6cc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff742dad0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff73eee50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 cr:s=0x55bff73e5190:op=0x55bff74292f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 cr:s=0x55bff73f3050:op=0x55bff73f7af0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 cr:s=0x55bff73fba60:op=0x55bff74230d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 cr:s=0x55bff7457930:op=0x55bff74236c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 cr:s=0x55bff73e0e60:op=0x55bff73e7330:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff742abb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is io blocked 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff73e2910:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 run: stack=0x55bff73ef490 is io blocked 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 cr:s=0x55bff73f0960:op=0x55bff73e0010:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 run: stack=0x55bff73f0960 is io blocked 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 cr:s=0x55bff7426880:op=0x55bff73e6cc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 run: stack=0x55bff7426880 is io blocked 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff742dad0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is io blocked 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff73eee50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 run: stack=0x55bff73e14a0 is io blocked 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 cr:s=0x55bff73e5190:op=0x55bff74292f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 run: stack=0x55bff73e5190 is io blocked 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 cr:s=0x55bff73f3050:op=0x55bff73f7af0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.799+0000 7fe839341900 20 run: stack=0x55bff73f3050 is io blocked 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.800+0000 7fe839341900 20 cr:s=0x55bff73fba60:op=0x55bff74230d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.800+0000 7fe839341900 20 run: stack=0x55bff73fba60 is io blocked 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.800+0000 7fe839341900 20 cr:s=0x55bff7457930:op=0x55bff74236c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.800+0000 7fe839341900 20 run: stack=0x55bff7457930 is io blocked 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.800+0000 7fe839341900 20 cr:s=0x55bff73e0e60:op=0x55bff73e7330:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.797 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.800+0000 7fe839341900 20 run: stack=0x55bff73e0e60 is io blocked 2026-03-20T11:46:42.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.801+0000 7fe839341900 20 cr:s=0x55bff73f0960:op=0x55bff73e0010:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.801+0000 7fe839341900 20 cr:s=0x55bff73f0960:op=0x55bff73e0010:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.801+0000 7fe839341900 20 cr:s=0x55bff73f0960:op=0x55bff73e0010:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.801+0000 7fe839341900 20 cr:s=0x55bff73f0960:op=0x55bff73e0010:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.801+0000 7fe839341900 15 stack 0x55bff73f0960 end 2026-03-20T11:46:42.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.801+0000 7fe839341900 20 run: stack=0x55bff73f0960 is done 2026-03-20T11:46:42.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.801+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e8f30:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:42.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.801+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.801+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff742abb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.801+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff742abb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.801+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff742abb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.801+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff742abb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.801+0000 7fe839341900 15 stack 0x55bff73f32f0 end 2026-03-20T11:46:42.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.801+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is done 2026-03-20T11:46:42.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.801+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e8f30:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:42.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.801+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff7457930:op=0x55bff74236c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff7457930:op=0x55bff74236c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff7457930:op=0x55bff74236c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.798 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff7457930:op=0x55bff74236c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 15 stack 0x55bff7457930 end 2026-03-20T11:46:42.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 run: stack=0x55bff7457930 is done 2026-03-20T11:46:42.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e8f30:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:42.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff73e0e60:op=0x55bff73e7330:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff73e0e60:op=0x55bff73e7330:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff73e2910:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff73e0e60:op=0x55bff73e7330:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff73e2910:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff73e0e60:op=0x55bff73e7330:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 15 stack 0x55bff73e0e60 end 2026-03-20T11:46:42.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 run: stack=0x55bff73e0e60 is done 2026-03-20T11:46:42.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff73e2910:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e8f30:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:42.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff73e2910:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 15 stack 0x55bff73ef490 end 2026-03-20T11:46:42.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 run: stack=0x55bff73ef490 is done 2026-03-20T11:46:42.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e8f30:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:42.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff73eee50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff73eee50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff73eee50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff73eee50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 15 stack 0x55bff73e14a0 end 2026-03-20T11:46:42.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 run: stack=0x55bff73e14a0 is done 2026-03-20T11:46:42.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e8f30:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:42.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff742dad0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff742dad0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff7426880:op=0x55bff73e6cc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff73e5190:op=0x55bff74292f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff742dad0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff7426880:op=0x55bff73e6cc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff73e5190:op=0x55bff74292f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff742dad0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 15 stack 0x55bff73e0d20 end 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is done 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff7426880:op=0x55bff73e6cc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff73e5190:op=0x55bff74292f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e8f30:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff7426880:op=0x55bff73e6cc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 15 stack 0x55bff7426880 end 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 run: stack=0x55bff7426880 is done 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff73e5190:op=0x55bff74292f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 15 stack 0x55bff73e5190 end 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 run: stack=0x55bff73e5190 is done 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e8f30:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e8f30:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff73f3050:op=0x55bff73f7af0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff73f3050:op=0x55bff73f7af0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff73f3050:op=0x55bff73f7af0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff73f3050:op=0x55bff73f7af0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 15 stack 0x55bff73f3050 end 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 run: stack=0x55bff73f3050 is done 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e8f30:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff73fba60:op=0x55bff74230d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff73fba60:op=0x55bff74230d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff73fba60:op=0x55bff74230d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff73fba60:op=0x55bff74230d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 15 stack 0x55bff73fba60 end 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 run: stack=0x55bff73fba60 is done 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e8f30:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e0010:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e0010:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 enqueued request req=0x55bff74298e0 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 req: 0x55bff74298e0 2026-03-20T11:46:42.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is io blocked 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe7ee7fc640 20 dequeued request req=0x55bff74298e0 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.802+0000 7fe7ee7fc640 20 RGWWQ: empty 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.803+0000 7fe839341900 20 cr:s=0x55bff73ec000:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.803+0000 7fe839341900 20 cr:s=0x55bff73ec000:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.803+0000 7fe839341900 20 cr:s=0x55bff73ec000:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.803+0000 7fe839341900 20 cr:s=0x55bff73ec000:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.803+0000 7fe839341900 20 cr:s=0x55bff73ec000:op=0x55bff7426af0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.803+0000 7fe839341900 15 stack 0x55bff73ec000 end 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.803+0000 7fe839341900 20 run: stack=0x55bff73ec000 is done 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.803+0000 7fe839341900 20 cr:s=0x55bff72fa2e0:op=0x55bff73ded40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.803+0000 7fe839341900 20 collect(): s=0x55bff72fa2e0 stack=0x55bff73ec000 is complete 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.803+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.803+0000 7fe839341900 10 trimming bilogs for bucket=:dipepu-19[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24]) markers=[,,,,,,,,00000000005.35.12,,], shards=11 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.803+0000 7fe839341900 20 cr:s=0x55bff72fa2e0:op=0x55bff742dad0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.803+0000 7fe839341900 10 trimming bilog shard 8 of :dipepu-19[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24]) at marker 00000000005.35.12 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.803+0000 7fe839341900 20 run: stack=0x55bff72fa2e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.803+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff742edc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.803+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff742edc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.803+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24.8 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.803+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is io blocked 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73e2f80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73e2f80:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f5c9b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 15 stack 0x55bff73ec910 end 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 run: stack=0x55bff73ec910 is done 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73dda70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 collect(): s=0x55bff74261a0 stack=0x55bff73ec910 is complete 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73dda70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 15 stack 0x55bff74261a0 end 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 stack->operate() returned ret=-22 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 run: stack=0x55bff74261a0 is done 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73dda70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 4 starting trim on bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff7429b00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff7429b00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 enqueued request req=0x55bff73ed4a0 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 req: 0x55bff73ed4a0 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 run: stack=0x55bff73ec910 is io blocked 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe7edffb640 20 dequeued request req=0x55bff73ed4a0 2026-03-20T11:46:42.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe7edffb640 20 RGWWQ: empty 2026-03-20T11:46:42.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe7edffb640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 : miss 2026-03-20T11:46:42.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe7edffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe7ee7fc640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 info.flags=0x11 2026-03-20T11:46:42.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe7ee7fc640 10 rgw async rados processor: moving a2.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 to cache LRU end 2026-03-20T11:46:42.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe7ee7fc640 10 rgw async rados processor: distributing notification oid=a2.rgw.control:notify.5 cni=[op: 0, obj: a2.rgw.meta:root:.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10, ofs0, ns] 2026-03-20T11:46:42.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe7edffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:42.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe7edffb640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 info.flags=0x17 2026-03-20T11:46:42.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe7edffb640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 to cache LRU end 2026-03-20T11:46:42.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe7edffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:42.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe7edffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:42.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe7edffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:46:42.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe7edffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-14 : miss 2026-03-20T11:46:42.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe7edffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff742edc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff742edc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff742edc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff742edc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 15 stack 0x55bff73e0d20 end 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is done 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 cr:s=0x55bff72fa2e0:op=0x55bff742dad0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 cr:s=0x55bff72fa2e0:op=0x55bff73ded40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 trim: trimmed bucket instance dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 15 stack 0x55bff72fa2e0 end 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 run: stack=0x55bff72fa2e0 is done 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff73ded40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 4 starting trim on bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff73e2f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff73e2f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 enqueued request req=0x55bff73da3f0 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 req: 0x55bff73da3f0 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is io blocked 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe7ed7fa640 20 dequeued request req=0x55bff73da3f0 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.804+0000 7fe7ed7fa640 20 RGWWQ: empty 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7ed7fa640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 : miss 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7ed7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7edffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7edffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-14 info.flags=0x0 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7edffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-14 to cache LRU end 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7edffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-14 : miss 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7edffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7fffff640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 343597383764 cookie 94282975270128 notifier 6505 bl.length()=565 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7fffff640 10 rgw watcher librados: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 info.flags=0x11 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7fffff640 10 rgw watcher librados: moving a2.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 to cache LRU end 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7edffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7edffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-14 info.flags=0x0 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7edffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-14 to cache LRU end 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7edffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19): adding source pipe: {s={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7edffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19): adding dest pipe: {s={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7edffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7edffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7edffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff7429b00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff7429b00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff7429b00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff7429b00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73dda70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7ed7fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7ed7fa640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 info.flags=0x17 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7ed7fa640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 to cache LRU end 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7ed7fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7ed7fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff6f5c9b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7ed7fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73dda70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 run: stack=0x55bff73ec910 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7ed7fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-36 : miss 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7ed7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.803 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 7e57463bc7c272261e62a9847a7f74a13f413544255fe691d0b05d6663e35f51 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114642Z 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:7e57463bc7c272261e62a9847a7f74a13f413544255fe691d0b05d6663e35f51 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 0002c6833c24604d584a4e3f4778a638bc397e4040332e228fe5d1d17a260141 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7ed7fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7ed7fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-36 info.flags=0x0 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7ed7fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-36 to cache LRU end 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7ed7fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-36 : miss 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7ed7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=0002c6833c24604d584a4e3f4778a638bc397e4040332e228fe5d1d17a260141 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114642Z 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7ee7fc640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-5[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10]): orig_sources= new_sources= 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7ee7fc640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-5[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10]): potential sources added=[] removed=[] 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=8, curl_handle=0x55bff73f1200 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7ee7fc640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-5[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10]): orig_dests= new_dests= 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7ee7fc640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-5[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10]): potential dests added=[] removed=[] 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 run: stack=0x55bff73e14a0 is io blocked 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e0010:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e0010:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e0010:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73e0010:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73f1910 req_data->id=8, curl_handle=0x55bff73f1200 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff742a9f0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff742a9f0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 enqueued request req=0x55bff6efaf40 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.805+0000 7fe839341900 20 req: 0x55bff6efaf40 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe7ecff9640 20 dequeued request req=0x55bff6efaf40 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe7ecff9640 20 RGWWQ: empty 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is io blocked 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe7ed7fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe7ed7fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-36 info.flags=0x0 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe7ed7fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-36 to cache LRU end 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe7ed7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41): adding source pipe: {s={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe7ed7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41): adding dest pipe: {s={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe7ed7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe7ed7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe7ed7fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff73e2f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff73e2f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff73e2f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff73e2f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff73ded40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff73e0010:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff73ded40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff73e2f80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff73e2f80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.805 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 0b25c3216169e646bc56e1ec77df32c3992dfae13f61798f0177b74bef97e93f 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114642Z 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:0b25c3216169e646bc56e1ec77df32c3992dfae13f61798f0177b74bef97e93f 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.806+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.807+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.807+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.807+0000 7fe7ecff9640 10 rgw async rados processor: removing a2.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 from cache 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.807+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.807+0000 7fe7ecff9640 10 rgw async rados processor: distributing notification oid=a2.rgw.control:notify.5 cni=[op: 1, obj: a2.rgw.meta:root:.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10, ofs0, ns] 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.807+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = eef4bb5910effa894e5cc9d9e08573bd94fdd2958bf5feccaa1e771432e53a62 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.807+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=eef4bb5910effa894e5cc9d9e08573bd94fdd2958bf5feccaa1e771432e53a62 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.807+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.807+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114642Z 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.807+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.807+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=9, curl_handle=0x55bff73e8f10 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.807+0000 7fe839341900 20 run: stack=0x55bff73ef490 is io blocked 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.807+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73e9620 req_data->id=9, curl_handle=0x55bff73e8f10 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.808+0000 7fe810ff9640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 343597383765 cookie 94282975270128 notifier 6505 bl.length()=205 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.808+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7428cc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7428cc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff742a9f0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7428cc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff742a9f0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7428cc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff742a9f0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7457310:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 15 stack 0x55bff742c160 end 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 20 run: stack=0x55bff742c160 is done 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff742a9f0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:42.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73d8af0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 20 collect(): s=0x55bff73dbd10 stack=0x55bff742c160 is complete 2026-03-20T11:46:42.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:42.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 10 trimming bilogs for bucket=:dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) markers=[00000000004.87.6,00000000002.126.1,00000000004.75.6,00000000002.34.1,00000000004.89.6,00000000002.87.1,00000000002.128.1,00000000004.94.6,00000000002.37.1,00000000002.73.1,00000000002.61.1], shards=11 2026-03-20T11:46:42.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 20 trim: trimmed bucket instance dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 2026-03-20T11:46:42.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 15 stack 0x55bff72f1ec0 end 2026-03-20T11:46:42.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is done 2026-03-20T11:46:42.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73eee50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 10 trimming bilog shard 0 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.87.6 2026-03-20T11:46:42.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 10 trimming bilog shard 1 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.126.1 2026-03-20T11:46:42.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 10 trimming bilog shard 2 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.75.6 2026-03-20T11:46:42.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 10 trimming bilog shard 3 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.34.1 2026-03-20T11:46:42.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 10 trimming bilog shard 4 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.89.6 2026-03-20T11:46:42.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 10 trimming bilog shard 5 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.87.1 2026-03-20T11:46:42.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 10 trimming bilog shard 6 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.128.1 2026-03-20T11:46:42.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 10 trimming bilog shard 7 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.94.6 2026-03-20T11:46:42.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 10 trimming bilog shard 8 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.37.1 2026-03-20T11:46:42.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 10 trimming bilog shard 9 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.73.1 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 10 trimming bilog shard 10 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.61.1 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 20 run: stack=0x55bff73dbd10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.809+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7455770:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 cr:s=0x55bff73e0e60:op=0x55bff73f6500:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 cr:s=0x55bff7457930:op=0x55bff742a9f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff73f5b10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 cr:s=0x55bff73f0960:op=0x55bff7428cc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff7426af0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 cr:s=0x55bff6f6e8e0:op=0x55bff74560c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73f6e50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7427dc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 cr:s=0x55bff72fa2e0:op=0x55bff73f3430:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff6f6ec50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 4 starting trim on bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7455770:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.0 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 run: stack=0x55bff742c160 is io blocked 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 cr:s=0x55bff73e0e60:op=0x55bff73f6500:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.1 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 run: stack=0x55bff73e0e60 is io blocked 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 cr:s=0x55bff7457930:op=0x55bff742a9f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.2 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 run: stack=0x55bff7457930 is io blocked 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff73f5b10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.3 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is io blocked 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 cr:s=0x55bff73f0960:op=0x55bff7428cc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.4 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 run: stack=0x55bff73f0960 is io blocked 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff7426af0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.5 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is io blocked 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 cr:s=0x55bff6f6e8e0:op=0x55bff74560c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.6 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 run: stack=0x55bff6f6e8e0 is io blocked 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73f6e50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.7 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 run: stack=0x55bff74261a0 is io blocked 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7427dc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.8 2026-03-20T11:46:42.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 run: stack=0x55bff73e9140 is io blocked 2026-03-20T11:46:42.809 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 cr:s=0x55bff72fa2e0:op=0x55bff73f3430:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.809 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.9 2026-03-20T11:46:42.809 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 run: stack=0x55bff72fa2e0 is io blocked 2026-03-20T11:46:42.809 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff6f6ec50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.809 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.10 2026-03-20T11:46:42.809 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 run: stack=0x55bff7432090 is io blocked 2026-03-20T11:46:42.809 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74230d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.809 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74230d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.809 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 enqueued request req=0x55bff6f651c0 2026-03-20T11:46:42.809 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.809 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 req: 0x55bff6f651c0 2026-03-20T11:46:42.809 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is io blocked 2026-03-20T11:46:42.809 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe7ec7f8640 20 dequeued request req=0x55bff6f651c0 2026-03-20T11:46:42.809 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe7ec7f8640 20 RGWWQ: empty 2026-03-20T11:46:42.809 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe7ec7f8640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 : miss 2026-03-20T11:46:42.809 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.810+0000 7fe7ec7f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.809 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 cr:s=0x55bff73f0960:op=0x55bff7428cc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.809 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 cr:s=0x55bff73f0960:op=0x55bff7428cc0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 15 stack 0x55bff73f0960 end 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 run: stack=0x55bff73f0960 is done 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73eee50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 run: stack=0x55bff73dbd10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff73f5b10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff73f5b10:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 15 stack 0x55bff73f32f0 end 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is done 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73eee50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 run: stack=0x55bff73dbd10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff7426af0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff7426af0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 15 stack 0x55bff73e87e0 end 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is done 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73f6e50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73f6e50:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 15 stack 0x55bff74261a0 end 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 run: stack=0x55bff74261a0 is done 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7427dc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7427dc0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 15 stack 0x55bff73e9140 end 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 run: stack=0x55bff73e9140 is done 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73eee50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73eee50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73eee50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 run: stack=0x55bff73dbd10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff6f6ec50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff6f6ec50:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 15 stack 0x55bff7432090 end 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 run: stack=0x55bff7432090 is done 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73eee50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe839341900 20 run: stack=0x55bff73dbd10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe7ec7f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe7ec7f8640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 info.flags=0x17 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe7ec7f8640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 to cache LRU end 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe7ec7f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe7ec7f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe7ec7f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe7ec7f8640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-22 : miss 2026-03-20T11:46:42.810 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.812+0000 7fe7ec7f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe7ec7f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe7ec7f8640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-22 info.flags=0x0 2026-03-20T11:46:42.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe7ec7f8640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-22 to cache LRU end 2026-03-20T11:46:42.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe7ec7f8640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-22 : miss 2026-03-20T11:46:42.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe7ec7f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7455770:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7455770:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 15 stack 0x55bff742c160 end 2026-03-20T11:46:42.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 run: stack=0x55bff742c160 is done 2026-03-20T11:46:42.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73eee50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 run: stack=0x55bff73dbd10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff7457930:op=0x55bff742a9f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff7457930:op=0x55bff742a9f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 15 stack 0x55bff7457930 end 2026-03-20T11:46:42.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.811 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 run: stack=0x55bff7457930 is done 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff73e0e60:op=0x55bff73f6500:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff73e0e60:op=0x55bff73f6500:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 15 stack 0x55bff73e0e60 end 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 run: stack=0x55bff73e0e60 is done 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73eee50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73eee50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 run: stack=0x55bff73dbd10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe7ec7f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe7ec7f8640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-22 info.flags=0x0 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe7ec7f8640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-22 to cache LRU end 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff6f6e8e0:op=0x55bff74560c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff6f6e8e0:op=0x55bff74560c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 15 stack 0x55bff6f6e8e0 end 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 run: stack=0x55bff6f6e8e0 is done 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73eee50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 run: stack=0x55bff73dbd10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff72fa2e0:op=0x55bff73f3430:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe7ec7f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27): adding source pipe: {s={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff72fa2e0:op=0x55bff73f3430:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 15 stack 0x55bff72fa2e0 end 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 run: stack=0x55bff72fa2e0 is done 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73eee50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff73dbd10:op=0x55bff73d8af0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 trim: trimmed bucket instance dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 15 stack 0x55bff73dbd10 end 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe7ec7f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27): adding dest pipe: {s={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 run: stack=0x55bff73dbd10 is done 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 4 starting trim on bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe7ec7f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 enqueued request req=0x55bff73f1540 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 req: 0x55bff73f1540 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe7ec7f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 run: stack=0x55bff742c160 is io blocked 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe7ec7f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe7ec7f8640 20 dequeued request req=0x55bff73f1540 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe7ec7f8640 20 RGWWQ: empty 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74230d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74230d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74230d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74230d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe7ec7f8640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 : miss 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7424ca0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff73eada0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff73eada0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe7ec7f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 9f398074e22f50b2d9a4f6984975af21f67cd7cc556afab6e93225b7536433f0 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114642Z 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:9f398074e22f50b2d9a4f6984975af21f67cd7cc556afab6e93225b7536433f0 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe7ec7f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 9e3cc3a4e2c86b6449777edbc7ef7f4cc8f133e3250cea176c88de813caabc32 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=9e3cc3a4e2c86b6449777edbc7ef7f4cc8f133e3250cea176c88de813caabc32 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114642Z 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe7ec7f8640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 info.flags=0x17 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe7ec7f8640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 to cache LRU end 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe7ec7f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe7ec7f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=10, curl_handle=0x55bff73ecfc0 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe839341900 20 run: stack=0x55bff7432090 is io blocked 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.813+0000 7fe7ec7f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe7ec7f8640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-25 : miss 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe7ec7f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe7e0fe1640 20 link_request req_data=0x7fe7c4004b10 req_data->id=10, curl_handle=0x55bff73ecfc0 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe7ec7f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe7ec7f8640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-25 info.flags=0x0 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe7ec7f8640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-25 to cache LRU end 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe7ec7f8640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-25 : miss 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe7ec7f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe7ec7f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe7ec7f8640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-25 info.flags=0x0 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe7ec7f8640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-25 to cache LRU end 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe7ec7f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30): adding source pipe: {s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe7ec7f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30): adding dest pipe: {s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe7ec7f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.813 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe7ec7f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe7ec7f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7457310:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 20 run: stack=0x55bff742c160 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff74230d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff74230d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 8085e4572725f7e4f9b059f9d6686f1e5b9e60ef620abdd8dd40625fee980110 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114642Z 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:8085e4572725f7e4f9b059f9d6686f1e5b9e60ef620abdd8dd40625fee980110 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 7ee5290687a7f1f5e06716d7dc829e1d13318c6c6df5d5ed37ff0d3e476afddd 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=7ee5290687a7f1f5e06716d7dc829e1d13318c6c6df5d5ed37ff0d3e476afddd 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114642Z 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=11, curl_handle=0x55bff73e4760 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.814+0000 7fe839341900 20 run: stack=0x55bff73e9140 is io blocked 2026-03-20T11:46:42.814 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.815+0000 7fe7e0fe1640 20 link_request req_data=0x7fe7d80062a0 req_data->id=11, curl_handle=0x55bff73e4760 2026-03-20T11:46:42.814 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:42.816+0000 7f126b1c6640 -1 req 18388388310864006262 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:42.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.849+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.849+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.849+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.849+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.849+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff6f5c9b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.849+0000 7fe839341900 15 stack 0x55bff73e14a0 end 2026-03-20T11:46:42.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.849+0000 7fe839341900 20 run: stack=0x55bff73e14a0 is done 2026-03-20T11:46:42.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.849+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73dda70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.849+0000 7fe839341900 20 collect(): s=0x55bff73ec910 stack=0x55bff73e14a0 is complete 2026-03-20T11:46:42.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.849+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:42.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.849+0000 7fe839341900 10 trimming bilogs for bucket=:dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) markers=[,,,,,00000000003.36.10,,,00000000011.47.6,00000000003.43.6,], shards=11 2026-03-20T11:46:42.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.849+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73d8af0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.849+0000 7fe839341900 10 trimming bilog shard 5 of :dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) at marker 00000000003.36.10 2026-03-20T11:46:42.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.849+0000 7fe839341900 10 trimming bilog shard 8 of :dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) at marker 00000000011.47.6 2026-03-20T11:46:42.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.849+0000 7fe839341900 10 trimming bilog shard 9 of :dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) at marker 00000000003.43.6 2026-03-20T11:46:42.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.849+0000 7fe839341900 20 run: stack=0x55bff73ec910 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.849+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff73f5aa0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.849+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73ef5d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.849+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff6f71500:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.849+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff73f5aa0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.849+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19.5 2026-03-20T11:46:42.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.849+0000 7fe839341900 20 run: stack=0x55bff73e14a0 is io blocked 2026-03-20T11:46:42.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.849+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73ef5d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.849+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19.8 2026-03-20T11:46:42.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.849+0000 7fe839341900 20 run: stack=0x55bff74261a0 is io blocked 2026-03-20T11:46:42.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.849+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff6f71500:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.849+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19.9 2026-03-20T11:46:42.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.849+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is io blocked 2026-03-20T11:46:42.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.851+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff73e2f80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.851+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff73e2f80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.851+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73ef5d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.851+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff73e2f80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.851+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73ef5d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.851+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff73e2f80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.851+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73ef5d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.851+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff73e0010:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.851+0000 7fe839341900 15 stack 0x55bff73ef490 end 2026-03-20T11:46:42.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.851+0000 7fe839341900 20 run: stack=0x55bff73ef490 is done 2026-03-20T11:46:42.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.851+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73ef5d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.851+0000 7fe839341900 15 stack 0x55bff74261a0 end 2026-03-20T11:46:42.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.851+0000 7fe839341900 20 run: stack=0x55bff74261a0 is done 2026-03-20T11:46:42.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.851+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff73ded40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.851+0000 7fe839341900 20 collect(): s=0x55bff73e0d20 stack=0x55bff73ef490 is complete 2026-03-20T11:46:42.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.851+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:42.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.851+0000 7fe839341900 10 trimming bilogs for bucket=:dipepu-36[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41]) markers=[,,,,,,,,,,00000000001.53.6], shards=11 2026-03-20T11:46:42.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.851+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73d8af0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.851+0000 7fe839341900 20 run: stack=0x55bff73ec910 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.851+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff6f5c9b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.851+0000 7fe839341900 10 trimming bilog shard 10 of :dipepu-36[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41]) at marker 00000000001.53.6 2026-03-20T11:46:42.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.851+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.851+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73fa1a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.851+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73fa1a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.851+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41.10 2026-03-20T11:46:42.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.851+0000 7fe839341900 20 run: stack=0x55bff74261a0 is io blocked 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff73f5aa0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff73f5aa0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff6f71500:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff73f5aa0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff6f71500:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff73f5aa0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 15 stack 0x55bff73e14a0 end 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 20 run: stack=0x55bff73e14a0 is done 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff6f71500:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73d8af0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 20 run: stack=0x55bff73ec910 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff6f71500:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 15 stack 0x55bff73e87e0 end 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is done 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73d8af0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73dda70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 20 trim: trimmed bucket instance dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 15 stack 0x55bff73ec910 end 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 20 run: stack=0x55bff73ec910 is done 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73dda70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 4 starting trim on bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f723f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f723f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 20 enqueued request req=0x55bff72eaa20 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 20 req: 0x55bff72eaa20 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe839341900 20 run: stack=0x55bff73ec910 is io blocked 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe7eb7f6640 20 dequeued request req=0x55bff72eaa20 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe7eb7f6640 20 RGWWQ: empty 2026-03-20T11:46:42.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe7eb7f6640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 : miss 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe7eb7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe7eb7f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe7eb7f6640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 info.flags=0x17 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe7eb7f6640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 to cache LRU end 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe7eb7f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe7eb7f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe7eb7f6640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe7eb7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe7eb7f6640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-35 : miss 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.852+0000 7fe7eb7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe7eb7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe7eb7f6640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-35 info.flags=0x0 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73fa1a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe7eb7f6640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-35 to cache LRU end 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73fa1a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73fa1a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73fa1a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 15 stack 0x55bff74261a0 end 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe7eb7f6640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-35 : miss 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 run: stack=0x55bff74261a0 is done 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff6f5c9b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe7eb7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff73ded40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 trim: trimmed bucket instance dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 15 stack 0x55bff73e0d20 end 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is done 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff6f71030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 4 starting trim on bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff742ef20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff742ef20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 enqueued request req=0x55bff7425b80 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 req: 0x55bff7425b80 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is io blocked 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe7eaff5640 20 dequeued request req=0x55bff7425b80 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe7eaff5640 20 RGWWQ: empty 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe7eaff5640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 : miss 2026-03-20T11:46:42.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe7eaff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe7eb7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe7eb7f6640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-35 info.flags=0x0 2026-03-20T11:46:42.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe7eb7f6640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-35 to cache LRU end 2026-03-20T11:46:42.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe7eb7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40): adding source pipe: {s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe7eb7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40): adding dest pipe: {s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe7eb7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe7eb7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe7eb7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:46:42.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f723f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f723f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f723f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f723f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73dda70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f5c9b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73dda70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 run: stack=0x55bff73ec910 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f70960:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f70960:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe7eaff5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe7eaff5640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 info.flags=0x17 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe7eaff5640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 to cache LRU end 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe7eaff5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe7eaff5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe7eaff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe7eaff5640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-26 : miss 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe7eaff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 5393d9e898cb38ddfb6e17c0e2126875b410602d7d78856eacb5bbe62e02243d 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114642Z 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:5393d9e898cb38ddfb6e17c0e2126875b410602d7d78856eacb5bbe62e02243d 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = ccbc0fcd74ece47bf560667c06fb9cc8939f1d028c920e9f9bbf0a6a5decd6d4 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=ccbc0fcd74ece47bf560667c06fb9cc8939f1d028c920e9f9bbf0a6a5decd6d4 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114642Z 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.853+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=12, curl_handle=0x55bff73e8f10 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.854+0000 7fe839341900 20 run: stack=0x55bff74261a0 is io blocked 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.854+0000 7fe7eaff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.854+0000 7fe7eaff5640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-26 info.flags=0x0 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.854+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73e5370 req_data->id=12, curl_handle=0x55bff73e8f10 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.854+0000 7fe7eaff5640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-26 to cache LRU end 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.854+0000 7fe7eaff5640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-26 : miss 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.854+0000 7fe7eaff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.854+0000 7fe7eaff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.854+0000 7fe7eaff5640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-26 info.flags=0x0 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.854+0000 7fe7eaff5640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-26 to cache LRU end 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.854+0000 7fe7eaff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31): adding source pipe: {s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.854+0000 7fe7eaff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31): adding dest pipe: {s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.854+0000 7fe7eaff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.854+0000 7fe7eaff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.854+0000 7fe7eaff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.854+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff742ef20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.854+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff742ef20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.854+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff742ef20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.854+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff742ef20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.854+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff6f71030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.854+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff7425330:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.854+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff6f71030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.854+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.854+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.854+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.854+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.854+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.854+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.855+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.855+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.855+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.855+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 3285fd3fb05918b4838c94d2d0cdccd89d190d05ed609a1d1199f62961a20f15 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.855+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114642Z 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:3285fd3fb05918b4838c94d2d0cdccd89d190d05ed609a1d1199f62961a20f15 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.855+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.855+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.855+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.855+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.855+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 04f69c8c2c747cf788707f65d778ef72d75d4b1eb485047dadfbf7a2f8311264 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.855+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=04f69c8c2c747cf788707f65d778ef72d75d4b1eb485047dadfbf7a2f8311264 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.855+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.855+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114642Z 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.855+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.855+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=13, curl_handle=0x55bff73f1200 2026-03-20T11:46:42.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.855+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is io blocked 2026-03-20T11:46:42.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.855+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73ed6d0 req_data->id=13, curl_handle=0x55bff73f1200 2026-03-20T11:46:42.854 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:42.856+0000 7f12af24e640 -1 req 14598256963558020141 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:42.854 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:42.856+0000 7f121d92b640 -1 req 4462710802413063525 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:42.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff74230d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff74230d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:42.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7457310:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe839341900 15 stack 0x55bff73e9140 end 2026-03-20T11:46:42.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe839341900 20 run: stack=0x55bff73e9140 is done 2026-03-20T11:46:42.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe839341900 20 collect(): s=0x55bff742c160 stack=0x55bff73e9140 is complete 2026-03-20T11:46:42.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:42.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe839341900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:42.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe839341900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:42.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:42.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe839341900 15 stack 0x55bff742c160 end 2026-03-20T11:46:42.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe839341900 20 stack->operate() returned ret=-22 2026-03-20T11:46:42.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe839341900 20 run: stack=0x55bff742c160 is done 2026-03-20T11:46:42.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:42.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe839341900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:42.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe839341900 4 starting trim on bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:42.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe839341900 20 enqueued request req=0x55bff73f1540 2026-03-20T11:46:42.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe839341900 20 req: 0x55bff73f1540 2026-03-20T11:46:42.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe839341900 20 run: stack=0x55bff742c160 is io blocked 2026-03-20T11:46:42.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe7ea7f4640 20 dequeued request req=0x55bff73f1540 2026-03-20T11:46:42.854 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe7ea7f4640 20 RGWWQ: empty 2026-03-20T11:46:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe7ea7f4640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 : miss 2026-03-20T11:46:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe7ea7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe7ea7f4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe7ea7f4640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 info.flags=0x17 2026-03-20T11:46:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe7ea7f4640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 to cache LRU end 2026-03-20T11:46:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe7ea7f4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe7ea7f4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe7ea7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe7ea7f4640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-51 : miss 2026-03-20T11:46:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe7ea7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe7ea7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe7ea7f4640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-51 info.flags=0x0 2026-03-20T11:46:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe7ea7f4640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-51 to cache LRU end 2026-03-20T11:46:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe7ea7f4640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-51 : miss 2026-03-20T11:46:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe7ea7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe7ea7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe7ea7f4640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-51 info.flags=0x0 2026-03-20T11:46:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe7ea7f4640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-51 to cache LRU end 2026-03-20T11:46:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe7ea7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11): adding source pipe: {s={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe7ea7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11): adding dest pipe: {s={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe7ea7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.857+0000 7fe7ea7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe7ea7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.855 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7457310:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 run: stack=0x55bff742c160 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff742eed0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff742eed0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = b2058a455f9eff4a0ba37958bb4372cfa6ba56cb1987a13f2167560408b361f6 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114642Z 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:b2058a455f9eff4a0ba37958bb4372cfa6ba56cb1987a13f2167560408b361f6 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = df91518c85850806ea9d700d00f00431e21a260cf08f37e3f6ef87b214d370a7 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=df91518c85850806ea9d700d00f00431e21a260cf08f37e3f6ef87b214d370a7 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114642Z 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=14, curl_handle=0x55bff73e4760 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 run: stack=0x55bff73e9140 is io blocked 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff73eada0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff73eada0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff73eada0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff73eada0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7424ca0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 15 stack 0x55bff7432090 end 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 run: stack=0x55bff7432090 is done 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 collect(): s=0x55bff72f1ec0 stack=0x55bff7432090 is complete 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 10 trimming bilogs for bucket=:dipepu-22[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27]) markers=[,,,,,,,,00000000005.29.12,,], shards=11 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff7424ca0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 10 trimming bilog shard 8 of :dipepu-22[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27]) at marker 00000000005.29.12 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7455770:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7455770:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27.8 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe839341900 20 run: stack=0x55bff7432090 is io blocked 2026-03-20T11:46:42.856 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.858+0000 7fe7e0fe1640 20 link_request req_data=0x55bff742f500 req_data->id=14, curl_handle=0x55bff73e4760 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7455770:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7455770:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7455770:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7455770:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe839341900 15 stack 0x55bff7432090 end 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe839341900 20 run: stack=0x55bff7432090 is done 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff7424ca0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe839341900 20 trim: trimmed bucket instance dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe839341900 15 stack 0x55bff72f1ec0 end 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is done 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe839341900 4 starting trim on bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5d8c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5d8c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe839341900 20 enqueued request req=0x55bff73e2170 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe839341900 20 req: 0x55bff73e2170 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is io blocked 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe7e9ff3640 20 dequeued request req=0x55bff73e2170 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe7e9ff3640 20 RGWWQ: empty 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe7e9ff3640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 : miss 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe7e9ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe7e9ff3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe7e9ff3640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 info.flags=0x17 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe7e9ff3640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 to cache LRU end 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe7e9ff3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe7e9ff3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe7e9ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe7e9ff3640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-55 : miss 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe7e9ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe7e9ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe7e9ff3640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-55 info.flags=0x0 2026-03-20T11:46:42.857 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe7e9ff3640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-55 to cache LRU end 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe7e9ff3640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-55 : miss 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.860+0000 7fe7e9ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe7e9ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe7e9ff3640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-55 info.flags=0x0 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe7e9ff3640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-55 to cache LRU end 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe7e9ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding source pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe7e9ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding dest pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe7e9ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe7e9ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe7e9ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5d8c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5d8c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5d8c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5d8c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7424ca0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff73eada0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff73eada0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:42.858 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:42.859 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:42.859 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.859 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.859 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.859 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.859 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.859 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:42.859 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = efe109652cae41548f1381fa74323a08b2b82b917fa00a02e2f1a01512abfbd0 2026-03-20T11:46:42.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:42.859 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114642Z 2026-03-20T11:46:42.859 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:42.859 INFO:teuthology.orchestra.run.vm02.stderr:efe109652cae41548f1381fa74323a08b2b82b917fa00a02e2f1a01512abfbd0 2026-03-20T11:46:42.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:42.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:42.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:42.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:42.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 718a506ba611eb6448c86f3a64131e92bf75f807f735c6025895aeb5cf9fe718 2026-03-20T11:46:42.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=718a506ba611eb6448c86f3a64131e92bf75f807f735c6025895aeb5cf9fe718 2026-03-20T11:46:42.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114642Z 2026-03-20T11:46:42.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:42.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=15, curl_handle=0x55bff73ecfc0 2026-03-20T11:46:42.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe839341900 20 run: stack=0x55bff7432090 is io blocked 2026-03-20T11:46:42.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.861+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73ec720 req_data->id=15, curl_handle=0x55bff73ecfc0 2026-03-20T11:46:42.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f70960:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f70960:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:42.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f5c9b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 15 stack 0x55bff74261a0 end 2026-03-20T11:46:42.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 run: stack=0x55bff74261a0 is done 2026-03-20T11:46:42.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73dda70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 collect(): s=0x55bff73ec910 stack=0x55bff74261a0 is complete 2026-03-20T11:46:42.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:42.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:42.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:42.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73dda70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:42.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 15 stack 0x55bff73ec910 end 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 stack->operate() returned ret=-22 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 run: stack=0x55bff73ec910 is done 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff7434900:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 4 starting trim on bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f5d8c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f5d8c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 enqueued request req=0x55bff72eaa20 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 req: 0x55bff72eaa20 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 run: stack=0x55bff73ec910 is io blocked 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe7e97f2640 20 dequeued request req=0x55bff72eaa20 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe7e97f2640 20 RGWWQ: empty 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe7e97f2640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 : miss 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff7425330:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 15 stack 0x55bff73e87e0 end 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is done 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe7e97f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff6f71030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 collect(): s=0x55bff73e0d20 stack=0x55bff73e87e0 is complete 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff6f71030:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 15 stack 0x55bff73e0d20 end 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 stack->operate() returned ret=-22 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is done 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff6f70fc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 4 starting trim on bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 enqueued request req=0x55bff7425b80 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 req: 0x55bff7425b80 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is io blocked 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe7e8ff1640 20 dequeued request req=0x55bff7425b80 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe7e8ff1640 20 RGWWQ: empty 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe7e8ff1640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 : miss 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe7e8ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe7e8ff1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe7e8ff1640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 info.flags=0x17 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe7e8ff1640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 to cache LRU end 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe7e8ff1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe7e8ff1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe7e8ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe7e8ff1640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-31 : miss 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe7e8ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe7e97f2640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:42.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe7e97f2640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 info.flags=0x17 2026-03-20T11:46:42.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe7e97f2640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 to cache LRU end 2026-03-20T11:46:42.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe7e97f2640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:42.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe7e97f2640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:42.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe7e97f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:42.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe7e97f2640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-28 : miss 2026-03-20T11:46:42.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.897+0000 7fe7e97f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe7e8ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe7e8ff1640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-31 info.flags=0x0 2026-03-20T11:46:42.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe7e8ff1640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-31 to cache LRU end 2026-03-20T11:46:42.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe7e8ff1640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-31 : miss 2026-03-20T11:46:42.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe7e8ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe7e97f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe7e97f2640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-28 info.flags=0x0 2026-03-20T11:46:42.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe7e97f2640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-28 to cache LRU end 2026-03-20T11:46:42.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe7e97f2640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-28 : miss 2026-03-20T11:46:42.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe7e97f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe7e8ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe7e8ff1640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-31 info.flags=0x0 2026-03-20T11:46:42.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe7e8ff1640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-31 to cache LRU end 2026-03-20T11:46:42.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe7e8ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding source pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe7e8ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding dest pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe7e8ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe7e8ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe7e8ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:42.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff6f70fc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe7e97f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe7e97f2640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-28 info.flags=0x0 2026-03-20T11:46:42.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe7e97f2640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-28 to cache LRU end 2026-03-20T11:46:42.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe7e97f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding source pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe7e97f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding dest pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe7e97f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe7e97f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe7e97f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff7425330:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff6f70fc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f5d8c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f5d8c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = befb0b4326746d5a2dc67eb4e5ab0bfb0f1fb4576b393130d821de7ac9f54fa7 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114642Z 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:befb0b4326746d5a2dc67eb4e5ab0bfb0f1fb4576b393130d821de7ac9f54fa7 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = ee6ca2f53090c3568bb3a2312c71c953062acbdc48e42e3ab504e0730ff3e00b 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=ee6ca2f53090c3568bb3a2312c71c953062acbdc48e42e3ab504e0730ff3e00b 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114642Z 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=16, curl_handle=0x55bff73f1200 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is io blocked 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f5d8c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f5d8c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.898+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff7434900:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.899+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f5c9b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.899+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff7434900:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.899+0000 7fe839341900 20 run: stack=0x55bff73ec910 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.899+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff7456900:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.899+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff7456900:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.899+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.899+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.899+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.899+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.899+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.899+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.899+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 33a0cd0cddc4e8c682a812b2c3daf27f4ef83b21fb498435d91932ad951f1e7a 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.899+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114642Z 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:33a0cd0cddc4e8c682a812b2c3daf27f4ef83b21fb498435d91932ad951f1e7a 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.899+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.899+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.899+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.899+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.899+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 87ebc77d7f15d6a4b89515483acaccc987bdd5c4a01e2009874fe7d200f28c33 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.899+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=87ebc77d7f15d6a4b89515483acaccc987bdd5c4a01e2009874fe7d200f28c33 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.899+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.899+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114642Z 2026-03-20T11:46:42.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.899+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:42.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.899+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=17, curl_handle=0x55bff73e8f10 2026-03-20T11:46:42.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.899+0000 7fe839341900 20 run: stack=0x55bff74261a0 is io blocked 2026-03-20T11:46:42.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.899+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73ed6d0 req_data->id=16, curl_handle=0x55bff73f1200 2026-03-20T11:46:42.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.899+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73e0e60 req_data->id=17, curl_handle=0x55bff73e8f10 2026-03-20T11:46:42.900 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:42.899+0000 7f11fc0e8640 -1 req 12027818667932324459 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:42.900 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:42.899+0000 7f124617c640 -1 req 4277897033676581649 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:42.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff742eed0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff742eed0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff742eed0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff742eed0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7457310:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 15 stack 0x55bff73e9140 end 2026-03-20T11:46:42.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 20 run: stack=0x55bff73e9140 is done 2026-03-20T11:46:42.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 20 collect(): s=0x55bff742c160 stack=0x55bff73e9140 is complete 2026-03-20T11:46:42.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:42.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 10 trimming bilogs for bucket=:dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) markers=[00000000002.62.6,,00000000002.48.6,,00000000002.70.6,,,00000000002.59.6,,,], shards=11 2026-03-20T11:46:42.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff73decd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 10 trimming bilog shard 0 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.62.6 2026-03-20T11:46:42.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 10 trimming bilog shard 2 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.48.6 2026-03-20T11:46:42.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 10 trimming bilog shard 4 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.70.6 2026-03-20T11:46:42.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 10 trimming bilog shard 7 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.59.6 2026-03-20T11:46:42.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 20 run: stack=0x55bff742c160 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff73f8e10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff6f6f3b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff742fc30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff73f3430:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff73f8e10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.0 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 20 run: stack=0x55bff73e9140 is io blocked 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff6f6f3b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.2 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 20 run: stack=0x55bff73e14a0 is io blocked 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff742fc30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.4 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 20 run: stack=0x55bff73ef490 is io blocked 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff73f3430:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.7 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.901+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is io blocked 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff6f6f3b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff6f6f3b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 15 stack 0x55bff73e14a0 end 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 run: stack=0x55bff73e14a0 is done 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff73decd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 run: stack=0x55bff742c160 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff73f8e10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff73f8e10:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 15 stack 0x55bff73e9140 end 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 run: stack=0x55bff73e9140 is done 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff73decd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 run: stack=0x55bff742c160 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff742fc30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff742fc30:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 15 stack 0x55bff73ef490 end 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 run: stack=0x55bff73ef490 is done 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff73decd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 run: stack=0x55bff742c160 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff73f3430:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff73f3430:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 15 stack 0x55bff73f32f0 end 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is done 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff73decd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 trim: trimmed bucket instance dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:42.902 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 15 stack 0x55bff742c160 end 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 run: stack=0x55bff742c160 is done 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 4 starting trim on bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 enqueued request req=0x55bff7423cb0 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 req: 0x55bff7423cb0 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe839341900 20 run: stack=0x55bff742c160 is io blocked 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe7e87f0640 20 dequeued request req=0x55bff7423cb0 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe7e87f0640 20 RGWWQ: empty 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe7e87f0640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 : miss 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.903+0000 7fe7e87f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff73eada0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe7e87f0640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe7e87f0640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 info.flags=0x17 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe7e87f0640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 to cache LRU end 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe7e87f0640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe7e87f0640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe7e87f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe7e87f0640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-39 : miss 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe7e87f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff73eada0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff73eada0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff73eada0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7424ca0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe839341900 15 stack 0x55bff7432090 end 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe839341900 20 run: stack=0x55bff7432090 is done 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe839341900 20 collect(): s=0x55bff72f1ec0 stack=0x55bff7432090 is complete 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe839341900 10 trimming bilogs for bucket=:dipepu-55[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15]) markers=[,,,,,00000000001.133.6,,,,,], shards=11 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f62040:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe839341900 10 trimming bilog shard 5 of :dipepu-55[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15]) at marker 00000000001.133.6 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff6f5d6f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff6f5d6f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15.5 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe839341900 20 run: stack=0x55bff7432090 is io blocked 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe7e87f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe7e87f0640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-39 info.flags=0x0 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe7e87f0640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-39 to cache LRU end 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe7e87f0640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-39 : miss 2026-03-20T11:46:42.903 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.904+0000 7fe7e87f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.905+0000 7fe7e87f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.905+0000 7fe7e87f0640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-39 info.flags=0x0 2026-03-20T11:46:42.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.905+0000 7fe7e87f0640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-39 to cache LRU end 2026-03-20T11:46:42.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.905+0000 7fe7e87f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding source pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.905+0000 7fe7e87f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding dest pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.905+0000 7fe7e87f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.905+0000 7fe7e87f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.905+0000 7fe7e87f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:42.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.905+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.905+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.905+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.905+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.905+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.905+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff7424ca0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.905+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.905+0000 7fe839341900 20 run: stack=0x55bff742c160 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.905+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff6f5d6f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.905+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff6f5d6f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.905+0000 7fe839341900 15 stack 0x55bff7432090 end 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.905+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.905+0000 7fe839341900 20 run: stack=0x55bff7432090 is done 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.905+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff73eada0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.905+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f62040:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.905+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff73eada0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.905+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.905+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.905+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 82ecb4d143ef98572d98e956a26a7e68c2f4aec21322e4ea6efa5b83550b8d0a 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114642Z 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:82ecb4d143ef98572d98e956a26a7e68c2f4aec21322e4ea6efa5b83550b8d0a 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = f9690defd700dedbafba1a530de58c72831e71745adfaa541cb109af53ade375 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=f9690defd700dedbafba1a530de58c72831e71745adfaa541cb109af53ade375 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114642Z 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=18, curl_handle=0x55bff73ecfc0 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is io blocked 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 20 trim: trimmed bucket instance dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 15 stack 0x55bff72f1ec0 end 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is done 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe7e0fe1640 20 link_request req_data=0x55bff74368b0 req_data->id=18, curl_handle=0x55bff73ecfc0 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.905 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 4 starting trim on bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:42.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 20 enqueued request req=0x55bff73e2170 2026-03-20T11:46:42.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 20 req: 0x55bff73e2170 2026-03-20T11:46:42.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is io blocked 2026-03-20T11:46:42.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.906+0000 7fe7e7fef640 20 dequeued request req=0x55bff73e2170 2026-03-20T11:46:42.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.907+0000 7fe7e7fef640 20 RGWWQ: empty 2026-03-20T11:46:42.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.907+0000 7fe7e7fef640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 : miss 2026-03-20T11:46:42.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.907+0000 7fe7e7fef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.907+0000 7fe7e7fef640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:46:42.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.907+0000 7fe7e7fef640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 info.flags=0x17 2026-03-20T11:46:42.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.907+0000 7fe7e7fef640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 to cache LRU end 2026-03-20T11:46:42.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.907+0000 7fe7e7fef640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:42.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.907+0000 7fe7e7fef640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:42.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.907+0000 7fe7e7fef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:42.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.907+0000 7fe7e7fef640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-8 : miss 2026-03-20T11:46:42.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.907+0000 7fe7e7fef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe7e7fef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe7e7fef640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-8 info.flags=0x0 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe7e7fef640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-8 to cache LRU end 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe7e7fef640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-8 : miss 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe7e7fef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe7e7fef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe7e7fef640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-8 info.flags=0x0 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe7e7fef640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-8 to cache LRU end 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe7e7fef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding source pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe7e7fef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding dest pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe7e7fef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe7e7fef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe7e7fef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff7457310:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7449b20:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7457310:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7457310:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 423bf9829bd8de999c8abaf64551651a57c601aec002105aac570451a102277b 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114642Z 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:423bf9829bd8de999c8abaf64551651a57c601aec002105aac570451a102277b 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 3fb1c7595923228d7c7fea55a3b2074b10a0e1784b2aa7a536968e2f7b944fe0 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=3fb1c7595923228d7c7fea55a3b2074b10a0e1784b2aa7a536968e2f7b944fe0 2026-03-20T11:46:42.907 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114642Z 2026-03-20T11:46:42.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:42.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=19, curl_handle=0x55bff73e4760 2026-03-20T11:46:42.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe7e0fe1640 20 link_request req_data=0x55bff6f63e70 req_data->id=19, curl_handle=0x55bff73e4760 2026-03-20T11:46:42.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.908+0000 7fe839341900 20 run: stack=0x55bff7432090 is io blocked 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff7425330:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 15 stack 0x55bff73e87e0 end 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is done 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff7456900:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff7456900:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff6f70fc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 collect(): s=0x55bff73e0d20 stack=0x55bff73e87e0 is complete 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff6f70fc0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 15 stack 0x55bff73e0d20 end 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 stack->operate() returned ret=-22 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is done 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f5c9b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 15 stack 0x55bff74261a0 end 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 run: stack=0x55bff74261a0 is done 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff7434900:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 collect(): s=0x55bff73ec910 stack=0x55bff74261a0 is complete 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff7434900:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 15 stack 0x55bff73ec910 end 2026-03-20T11:46:42.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 stack->operate() returned ret=-22 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 run: stack=0x55bff73ec910 is done 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff6f70f90:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 4 starting trim on bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff7435cb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 4 starting trim on bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff7435cb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 enqueued request req=0x55bff73ec720 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 req: 0x55bff73ec720 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is io blocked 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 enqueued request req=0x55bff73da3f0 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 req: 0x55bff73ec720 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 req: 0x55bff73da3f0 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe839341900 20 run: stack=0x55bff73ec910 is io blocked 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe7e6fed640 20 dequeued request req=0x55bff73ec720 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe7e6fed640 20 RGWWQ: 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe7e6fed640 20 req: 0x55bff73da3f0 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe7e6fed640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 : miss 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe7e6fed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe7e77ee640 20 dequeued request req=0x55bff73da3f0 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe7e77ee640 20 RGWWQ: empty 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe7e77ee640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 : miss 2026-03-20T11:46:42.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe7e77ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe7e6fed640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe7e6fed640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 info.flags=0x17 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe7e6fed640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 to cache LRU end 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe7e6fed640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe7e6fed640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe7e6fed640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe7e6fed640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-16 : miss 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe7e6fed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.940+0000 7fe7e77ee640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe7e77ee640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 info.flags=0x17 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe7e77ee640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 to cache LRU end 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe7e77ee640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe7e77ee640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe7e77ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe7e77ee640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-43 : miss 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe7e77ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe7e6fed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe7e6fed640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-16 info.flags=0x0 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe7e6fed640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-16 to cache LRU end 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe7e6fed640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-16 : miss 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe7e6fed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe7e77ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe7e77ee640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-43 info.flags=0x0 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe7e77ee640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-43 to cache LRU end 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe7e77ee640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-43 : miss 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe7e77ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe7e77ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe7e77ee640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-43 info.flags=0x0 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe7e77ee640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-43 to cache LRU end 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe7e77ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding source pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe7e77ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding dest pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe7e77ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe7e77ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe7e77ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f5c9b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe839341900 20 run: stack=0x55bff73ec910 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:42.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.941+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = d02b6a27e1bdeab2e4210fffe718659958719d629e26c00dafe3e76d54404987 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114642Z 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:d02b6a27e1bdeab2e4210fffe718659958719d629e26c00dafe3e76d54404987 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 23553b05805cada268da03103ef1793be65458924044a445d2fe759a058ec150 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=23553b05805cada268da03103ef1793be65458924044a445d2fe759a058ec150 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114642Z 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=20, curl_handle=0x55bff73e8f10 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 20 run: stack=0x55bff74261a0 is io blocked 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe7e6fed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe7e6fed640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-16 info.flags=0x0 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe7e6fed640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-16 to cache LRU end 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe7e6fed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding source pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe7e6fed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding dest pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73e0e60 req_data->id=20, curl_handle=0x55bff73e8f10 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe7e6fed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe7e6fed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe7e6fed640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff7435cb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff7435cb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff7435cb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff7435cb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff6f70f90:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff74252c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff6f70f90:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff7435cb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff7435cb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.942+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:42.941 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:42.942+0000 7f12cb286640 -1 req 12216253345772829925 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.943+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.943+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.943+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.943+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 2d74fe089c8b7c71f11f593b4523d44e529c20e1bc1188a6ab49aa286772f17e 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.943+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114642Z 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:2d74fe089c8b7c71f11f593b4523d44e529c20e1bc1188a6ab49aa286772f17e 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.943+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.943+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.943+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.943+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.943+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 0212bea223ca3e846ac320d5d87034e4d9505d6e192c0e18ea0b72e0d559553a 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.943+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=0212bea223ca3e846ac320d5d87034e4d9505d6e192c0e18ea0b72e0d559553a 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.943+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.943+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114642Z 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.943+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.943+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=21, curl_handle=0x55bff73f1200 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.943+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is io blocked 2026-03-20T11:46:42.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.943+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73ed6d0 req_data->id=21, curl_handle=0x55bff73f1200 2026-03-20T11:46:42.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.949+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff73eada0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.949+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff73eada0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.949+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff73eada0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.949+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff73eada0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.949+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff7424ca0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.949+0000 7fe839341900 15 stack 0x55bff73f32f0 end 2026-03-20T11:46:42.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.949+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is done 2026-03-20T11:46:42.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.949+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.949+0000 7fe839341900 20 collect(): s=0x55bff742c160 stack=0x55bff73f32f0 is complete 2026-03-20T11:46:42.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.949+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:42.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.949+0000 7fe839341900 10 trimming bilogs for bucket=:dipepu-39[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44]) markers=[,,,,,,,,00000000001.18.6,,], shards=11 2026-03-20T11:46:42.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.949+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff73e7a00:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.949+0000 7fe839341900 10 trimming bilog shard 8 of :dipepu-39[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44]) at marker 00000000001.18.6 2026-03-20T11:46:42.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.949+0000 7fe839341900 20 run: stack=0x55bff742c160 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.949+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff7456460:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.949+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff7456460:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.949+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44.8 2026-03-20T11:46:42.946 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.949+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is io blocked 2026-03-20T11:46:42.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff7456460:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff7456460:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe839341900 15 stack 0x55bff73f32f0 end 2026-03-20T11:46:42.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is done 2026-03-20T11:46:42.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff73e7a00:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe839341900 20 trim: trimmed bucket instance dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:42.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe839341900 15 stack 0x55bff742c160 end 2026-03-20T11:46:42.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe839341900 20 run: stack=0x55bff742c160 is done 2026-03-20T11:46:42.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:42.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe839341900 4 starting trim on bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:42.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe839341900 20 enqueued request req=0x55bff7423cb0 2026-03-20T11:46:42.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe839341900 20 req: 0x55bff7423cb0 2026-03-20T11:46:42.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe839341900 20 run: stack=0x55bff742c160 is io blocked 2026-03-20T11:46:42.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe7e67ec640 20 dequeued request req=0x55bff7423cb0 2026-03-20T11:46:42.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe7e67ec640 20 RGWWQ: empty 2026-03-20T11:46:42.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe7e67ec640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 : miss 2026-03-20T11:46:42.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe7e67ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.947 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe7e67ec640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe7e67ec640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 info.flags=0x17 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe7e67ec640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 to cache LRU end 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe7e67ec640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe7e67ec640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe7e67ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe7e67ec640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-10 : miss 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.950+0000 7fe7e67ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7457310:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe7e67ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe7e67ec640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-10 info.flags=0x0 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe7e67ec640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-10 to cache LRU end 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe7e67ec640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-10 : miss 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe7e67ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7457310:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7457310:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7457310:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7449b20:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 15 stack 0x55bff7432090 end 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 run: stack=0x55bff7432090 is done 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 collect(): s=0x55bff72f1ec0 stack=0x55bff7432090 is complete 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 10 trimming bilogs for bucket=:dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) markers=[,00000000002.13.6,,00000000001.5.6,,00000000001.2.6,00000000002.13.6,00000000001.11.6,00000000001.4.6,00000000001.12.6,], shards=11 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff7424ca0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 10 trimming bilog shard 1 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000002.13.6 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 10 trimming bilog shard 3 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.5.6 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 10 trimming bilog shard 5 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.2.6 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 10 trimming bilog shard 6 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000002.13.6 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 10 trimming bilog shard 7 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.11.6 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 10 trimming bilog shard 8 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.4.6 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 10 trimming bilog shard 9 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.12.6 2026-03-20T11:46:42.948 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff73f7200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff7430b10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff6f633a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff6f61180:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff73dfd30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 cr:s=0x55bff73f0960:op=0x55bff74233a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff6f61c80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff73f7200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.1 2026-03-20T11:46:42.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 run: stack=0x55bff7432090 is io blocked 2026-03-20T11:46:42.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff7430b10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.3 2026-03-20T11:46:42.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is io blocked 2026-03-20T11:46:42.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff6f633a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.5 2026-03-20T11:46:42.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 run: stack=0x55bff73ef490 is io blocked 2026-03-20T11:46:42.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff6f61180:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.6 2026-03-20T11:46:42.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 run: stack=0x55bff73e9140 is io blocked 2026-03-20T11:46:42.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff73dfd30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.7 2026-03-20T11:46:42.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 run: stack=0x55bff73e14a0 is io blocked 2026-03-20T11:46:42.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 cr:s=0x55bff73f0960:op=0x55bff74233a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.8 2026-03-20T11:46:42.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 run: stack=0x55bff73f0960 is io blocked 2026-03-20T11:46:42.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff6f61c80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.9 2026-03-20T11:46:42.949 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.951+0000 7fe839341900 20 run: stack=0x55bff7455ec0 is io blocked 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.952+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff7430b10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.952+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff7430b10:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.952+0000 7fe839341900 15 stack 0x55bff73f32f0 end 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.952+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.952+0000 7fe7e67ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.952+0000 7fe7e67ec640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-10 info.flags=0x0 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.952+0000 7fe7e67ec640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-10 to cache LRU end 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.952+0000 7fe7e67ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding source pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.952+0000 7fe7e67ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding dest pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.952+0000 7fe7e67ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.952+0000 7fe7e67ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.952+0000 7fe7e67ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.952+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is done 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.952+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff7424ca0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.952+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.952+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.952+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.952+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.952+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.952+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff7457310:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 20 run: stack=0x55bff742c160 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff6f60300:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff6f60300:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:42.950 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 4b1d24a66ee709166b756bd1a46ac176bb24de34a01975f1fd5f0d5f1e5aebf6 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114642Z 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:4b1d24a66ee709166b756bd1a46ac176bb24de34a01975f1fd5f0d5f1e5aebf6 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 7e0e7dc7e779b071efae15abb6268408d2dd277d24501cee048ea87cd2559047 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=7e0e7dc7e779b071efae15abb6268408d2dd277d24501cee048ea87cd2559047 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114642Z 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=22, curl_handle=0x55bff73e4760 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is io blocked 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 20 cr:s=0x55bff73f0960:op=0x55bff74233a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe7e0fe1640 20 link_request req_data=0x55bff6f63e70 req_data->id=22, curl_handle=0x55bff73e4760 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 20 cr:s=0x55bff73f0960:op=0x55bff74233a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 15 stack 0x55bff73f0960 end 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 20 run: stack=0x55bff73f0960 is done 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff7424ca0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.953+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff6f633a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff6f633a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 15 stack 0x55bff73ef490 end 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 run: stack=0x55bff73ef490 is done 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff7424ca0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff73f7200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff73f7200:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 15 stack 0x55bff7432090 end 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 run: stack=0x55bff7432090 is done 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff7424ca0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff73dfd30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff73dfd30:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 15 stack 0x55bff73e14a0 end 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 run: stack=0x55bff73e14a0 is done 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff6f61c80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff6f61c80:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 15 stack 0x55bff7455ec0 end 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 run: stack=0x55bff7455ec0 is done 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff7424ca0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff7424ca0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff6f61180:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff6f61180:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 15 stack 0x55bff73e9140 end 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 run: stack=0x55bff73e9140 is done 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff7424ca0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 trim: trimmed bucket instance dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 15 stack 0x55bff72f1ec0 end 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is done 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 4 starting trim on bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.952 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 enqueued request req=0x55bff73f0f70 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 req: 0x55bff73f0f70 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is io blocked 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe7e5feb640 20 dequeued request req=0x55bff73f0f70 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe7e5feb640 20 RGWWQ: empty 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe7e5feb640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 : miss 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.954+0000 7fe7e5feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe7e5feb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe7e5feb640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 info.flags=0x17 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe7e5feb640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 to cache LRU end 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe7e5feb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe7e5feb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe7e5feb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe7e5feb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-3 : miss 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe7e5feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe7e5feb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe7e5feb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-3 info.flags=0x0 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe7e5feb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-3 to cache LRU end 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe7e5feb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-3 : miss 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe7e5feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe7e5feb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe7e5feb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-3 info.flags=0x0 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe7e5feb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-3 to cache LRU end 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe7e5feb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding source pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe7e5feb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding dest pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe7e5feb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe7e5feb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe7e5feb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff6f6f6c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7455770:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7455770:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.953 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = bb97e0e35131f984cd276a7e70a022f6c3315f1292211e7324eb1456888e0aa8 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114642Z 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:bb97e0e35131f984cd276a7e70a022f6c3315f1292211e7324eb1456888e0aa8 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 90e9660a085a651c7782debda0df2bde166a9aad0f943e4cc55a204630561484 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=90e9660a085a651c7782debda0df2bde166a9aad0f943e4cc55a204630561484 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114642Z 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=23, curl_handle=0x55bff73ecfc0 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe839341900 20 run: stack=0x55bff73e9140 is io blocked 2026-03-20T11:46:42.954 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.955+0000 7fe7e0fe1640 20 link_request req_data=0x55bff74368b0 req_data->id=23, curl_handle=0x55bff73ecfc0 2026-03-20T11:46:42.954 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:42.956+0000 7f12839f7640 -1 req 17156151644611939350 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:42.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:42.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f5c9b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe839341900 15 stack 0x55bff74261a0 end 2026-03-20T11:46:42.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe839341900 20 run: stack=0x55bff74261a0 is done 2026-03-20T11:46:42.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe839341900 20 collect(): s=0x55bff73ec910 stack=0x55bff74261a0 is complete 2026-03-20T11:46:42.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:42.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe839341900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:42.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe839341900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:42.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:42.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe839341900 15 stack 0x55bff73ec910 end 2026-03-20T11:46:42.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe839341900 20 stack->operate() returned ret=-22 2026-03-20T11:46:42.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe839341900 20 run: stack=0x55bff73ec910 is done 2026-03-20T11:46:42.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:42.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe839341900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:42.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe839341900 4 starting trim on bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:42.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff74303b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff74303b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe839341900 20 enqueued request req=0x55bff73da3f0 2026-03-20T11:46:42.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe839341900 20 req: 0x55bff73da3f0 2026-03-20T11:46:42.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe839341900 20 run: stack=0x55bff73ec910 is io blocked 2026-03-20T11:46:42.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe7e57ea640 20 dequeued request req=0x55bff73da3f0 2026-03-20T11:46:42.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe7e57ea640 20 RGWWQ: empty 2026-03-20T11:46:42.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe7e57ea640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 : miss 2026-03-20T11:46:42.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe7e57ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe7e57ea640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:46:42.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe7e57ea640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 info.flags=0x17 2026-03-20T11:46:42.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe7e57ea640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 to cache LRU end 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe7e57ea640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe7e57ea640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe7e57ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe7e57ea640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-9 : miss 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe7e57ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe7e57ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe7e57ea640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-9 info.flags=0x0 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe7e57ea640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-9 to cache LRU end 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe7e57ea640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-9 : miss 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.983+0000 7fe7e57ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe7e57ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe7e57ea640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-9 info.flags=0x0 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe7e57ea640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-9 to cache LRU end 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe7e57ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding source pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe7e57ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding dest pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe7e57ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe7e57ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe7e57ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff74303b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff74303b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff74303b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff74303b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f5c9b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 20 run: stack=0x55bff73ec910 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.982 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.983 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:42.983 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 4b93c171f423958a92b541545ccd223c5a109164b05250c22b387d588bdb763b 2026-03-20T11:46:42.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:42.983 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114642Z 2026-03-20T11:46:42.983 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:42.983 INFO:teuthology.orchestra.run.vm02.stderr:4b93c171f423958a92b541545ccd223c5a109164b05250c22b387d588bdb763b 2026-03-20T11:46:42.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:42.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:42.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:42.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:42.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 82a2f7d417e603f61b8d3da90d9bd9f82e5ddcbcabd7fbe5607e7b09632df773 2026-03-20T11:46:42.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=82a2f7d417e603f61b8d3da90d9bd9f82e5ddcbcabd7fbe5607e7b09632df773 2026-03-20T11:46:42.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114642Z 2026-03-20T11:46:42.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:42.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=24, curl_handle=0x55bff73e8f10 2026-03-20T11:46:42.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe839341900 20 run: stack=0x55bff74261a0 is io blocked 2026-03-20T11:46:42.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.984+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73e0e60 req_data->id=24, curl_handle=0x55bff73e8f10 2026-03-20T11:46:42.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.986+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff7435cb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.986+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff7435cb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.986+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff7435cb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.986+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff7435cb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.986+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff74252c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.986+0000 7fe839341900 15 stack 0x55bff73e87e0 end 2026-03-20T11:46:42.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.986+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is done 2026-03-20T11:46:42.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.986+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff6f70f90:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.986+0000 7fe839341900 20 collect(): s=0x55bff73e0d20 stack=0x55bff73e87e0 is complete 2026-03-20T11:46:42.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.986+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:42.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.986+0000 7fe839341900 10 trimming bilogs for bucket=:dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) markers=[,,,,,00000000002.22.11,,,00000000031.119.12,,], shards=11 2026-03-20T11:46:42.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.986+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff7424ca0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.986+0000 7fe839341900 10 trimming bilog shard 5 of :dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) at marker 00000000002.22.11 2026-03-20T11:46:42.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.986+0000 7fe839341900 10 trimming bilog shard 8 of :dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) at marker 00000000031.119.12 2026-03-20T11:46:42.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.986+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.986+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff73dfd30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.986+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff6f5d100:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.986+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff73dfd30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.986+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21.5 2026-03-20T11:46:42.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.986+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is io blocked 2026-03-20T11:46:42.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.986+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff6f5d100:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.986+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21.8 2026-03-20T11:46:42.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.986+0000 7fe839341900 20 run: stack=0x55bff7455ec0 is io blocked 2026-03-20T11:46:42.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.987+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff73dfd30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.987+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff73dfd30:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.987+0000 7fe839341900 15 stack 0x55bff73e87e0 end 2026-03-20T11:46:42.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.987+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.987+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is done 2026-03-20T11:46:42.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.987+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff7424ca0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.987+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff6f5d100:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff6f5d100:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe839341900 15 stack 0x55bff7455ec0 end 2026-03-20T11:46:42.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe839341900 20 run: stack=0x55bff7455ec0 is done 2026-03-20T11:46:42.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff7424ca0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff6f70f90:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe839341900 20 trim: trimmed bucket instance dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:42.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe839341900 15 stack 0x55bff73e0d20 end 2026-03-20T11:46:42.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is done 2026-03-20T11:46:42.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:42.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff6f70f90:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe839341900 4 starting trim on bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:42.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff74303b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff74303b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe839341900 20 enqueued request req=0x55bff73e2170 2026-03-20T11:46:42.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe839341900 20 req: 0x55bff73e2170 2026-03-20T11:46:42.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe7e4fe9640 20 dequeued request req=0x55bff73e2170 2026-03-20T11:46:42.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe7e4fe9640 20 RGWWQ: empty 2026-03-20T11:46:42.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe7e4fe9640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 : miss 2026-03-20T11:46:42.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe7e4fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is io blocked 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe7e4fe9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe7e4fe9640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 info.flags=0x17 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe7e4fe9640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 to cache LRU end 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe7e4fe9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe7e4fe9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe7e4fe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe7e4fe9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-38 : miss 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe7e4fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe7e4fe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe7e4fe9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-38 info.flags=0x0 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe7e4fe9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-38 to cache LRU end 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe7e4fe9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-38 : miss 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.988+0000 7fe7e4fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe7e4fe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe7e4fe9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-38 info.flags=0x0 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe7e4fe9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-38 to cache LRU end 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe7e4fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding source pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe7e4fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding dest pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe7e4fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe7e4fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe7e4fe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff74303b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff74303b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff74303b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff74303b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff6f70f90:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff7456250:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff6f70f90:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff74252c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff74252c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.986 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:42.987 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:42.987 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:42.987 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.987 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.987 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.987 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.987 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.987 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:42.987 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 456fca958c68b454fb4d8010e51f9a7523ef328e9ff7f4f09d9e35b9f47a74f3 2026-03-20T11:46:42.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:42.987 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114642Z 2026-03-20T11:46:42.987 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:42.987 INFO:teuthology.orchestra.run.vm02.stderr:456fca958c68b454fb4d8010e51f9a7523ef328e9ff7f4f09d9e35b9f47a74f3 2026-03-20T11:46:42.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:42.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:42.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:42.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:42.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = b0ddfdc1f8d5767815f13d7dd9b62b339137954bae25588898b2082be6d25f05 2026-03-20T11:46:42.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b0ddfdc1f8d5767815f13d7dd9b62b339137954bae25588898b2082be6d25f05 2026-03-20T11:46:42.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114642Z 2026-03-20T11:46:42.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:42.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=25, curl_handle=0x55bff73f1200 2026-03-20T11:46:42.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe839341900 20 run: stack=0x55bff7455ec0 is io blocked 2026-03-20T11:46:42.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.989+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73ed6d0 req_data->id=25, curl_handle=0x55bff73f1200 2026-03-20T11:46:42.993 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff6f60300:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff6f60300:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7455770:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7455770:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff6f60300:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff6f6f6c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 15 stack 0x55bff73e9140 end 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 run: stack=0x55bff73e9140 is done 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff6f60300:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 collect(): s=0x55bff72f1ec0 stack=0x55bff73e9140 is complete 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 15 stack 0x55bff72f1ec0 end 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 stack->operate() returned ret=-22 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is done 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff7457310:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 15 stack 0x55bff73f32f0 end 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is done 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 collect(): s=0x55bff742c160 stack=0x55bff73f32f0 is complete 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 10 trimming bilogs for bucket=:dipepu-10[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15]) markers=[,00000000002.9.6,,,,,,,,,], shards=11 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 4 starting trim on bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7457310:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 10 trimming bilog shard 1 of :dipepu-10[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15]) at marker 00000000002.9.6 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 run: stack=0x55bff742c160 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f6f570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff74233a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f6f570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.994 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 enqueued request req=0x55bff73ed4a0 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 req: 0x55bff73ed4a0 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe7e47e8640 20 dequeued request req=0x55bff73ed4a0 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe7e47e8640 20 RGWWQ: empty 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe7e47e8640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 : miss 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe7e47e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is io blocked 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff74233a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15.1 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is io blocked 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe7e47e8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe7e47e8640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 info.flags=0x17 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe7e47e8640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 to cache LRU end 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe7e47e8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe7e47e8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe7e47e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe7e47e8640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-15 : miss 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.997+0000 7fe7e47e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe7e47e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe7e47e8640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-15 info.flags=0x0 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe7e47e8640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-15 to cache LRU end 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe7e47e8640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-15 : miss 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe7e47e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe7e47e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe7e47e8640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-15 info.flags=0x0 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe7e47e8640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-15 to cache LRU end 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe7e47e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding source pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe7e47e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding dest pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe7e47e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe7e47e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe7e47e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f6f570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f6f570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f6f570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f6f570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.995 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff744a6a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7455770:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7455770:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:42.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:42.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.996 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.996 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.996 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.996 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.996 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:42 +0000 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114642Z 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = ca3e8fc29aa10356efa96ccf4e5d95fad16c8525535145a28cad3d5039c31c36 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114642Z 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:ca3e8fc29aa10356efa96ccf4e5d95fad16c8525535145a28cad3d5039c31c36 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 701092ce3ba7463a59d49aea31a48eec0db5fe2cd5f5cd0bed74df23b1292f4c 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=701092ce3ba7463a59d49aea31a48eec0db5fe2cd5f5cd0bed74df23b1292f4c 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114642Z 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=26, curl_handle=0x55bff73ecfc0 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 run: stack=0x55bff73e9140 is io blocked 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe7e0fe1640 20 link_request req_data=0x55bff74368b0 req_data->id=26, curl_handle=0x55bff73ecfc0 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff74233a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff74233a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 15 stack 0x55bff73f32f0 end 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is done 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7457310:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 trim: trimmed bucket instance dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 15 stack 0x55bff742c160 end 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 run: stack=0x55bff742c160 is done 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 4 starting trim on bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff6f6f570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff6f6f570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 enqueued request req=0x55bff73ec720 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 req: 0x55bff73ec720 2026-03-20T11:46:42.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe839341900 20 run: stack=0x55bff742c160 is io blocked 2026-03-20T11:46:42.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe7fcff9640 20 dequeued request req=0x55bff73ec720 2026-03-20T11:46:42.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe7fcff9640 20 RGWWQ: empty 2026-03-20T11:46:42.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe7fcff9640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 : miss 2026-03-20T11:46:42.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.998+0000 7fe7fcff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.999+0000 7fe7fcff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:42.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.999+0000 7fe7fcff9640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 info.flags=0x17 2026-03-20T11:46:42.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.999+0000 7fe7fcff9640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 to cache LRU end 2026-03-20T11:46:42.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.999+0000 7fe7fcff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:42.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.999+0000 7fe7fcff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:42.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.999+0000 7fe7fcff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:42.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.999+0000 7fe7fcff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-13 : miss 2026-03-20T11:46:42.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.999+0000 7fe7fcff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.999+0000 7fe7fcff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.999+0000 7fe7fcff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-13 info.flags=0x0 2026-03-20T11:46:42.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.999+0000 7fe7fcff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-13 to cache LRU end 2026-03-20T11:46:42.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.999+0000 7fe7fcff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-13 : miss 2026-03-20T11:46:42.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:42.999+0000 7fe7fcff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:42.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe7fcff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:42.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe7fcff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-13 info.flags=0x0 2026-03-20T11:46:42.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe7fcff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-13 to cache LRU end 2026-03-20T11:46:42.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe7fcff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18): adding source pipe: {s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.998 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe7fcff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18): adding dest pipe: {s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe7fcff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe7fcff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe7fcff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff6f6f570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff6f6f570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff6f6f570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff6f6f570:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff7430340:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 20 run: stack=0x55bff742c160 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff7435c00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff7435c00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 22aac1ff2bf1d867cf525d0082e9d16f915a61be47376fd99261b92137b814cc 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114643Z 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:22aac1ff2bf1d867cf525d0082e9d16f915a61be47376fd99261b92137b814cc 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = f4c72178d39248fa5e7d9cd0cddd529ba9a50f23d316d2ccb2472c204fa31efb 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=f4c72178d39248fa5e7d9cd0cddd529ba9a50f23d316d2ccb2472c204fa31efb 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114643Z 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=27, curl_handle=0x55bff73e4760 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is io blocked 2026-03-20T11:46:42.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.000+0000 7fe7e0fe1640 20 link_request req_data=0x55bff6f63e70 req_data->id=27, curl_handle=0x55bff73e4760 2026-03-20T11:46:43.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.027+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.027+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.027+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.027+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.027+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f5c9b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.027+0000 7fe839341900 15 stack 0x55bff74261a0 end 2026-03-20T11:46:43.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.027+0000 7fe839341900 20 run: stack=0x55bff74261a0 is done 2026-03-20T11:46:43.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.027+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.027+0000 7fe839341900 20 collect(): s=0x55bff73ec910 stack=0x55bff74261a0 is complete 2026-03-20T11:46:43.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.027+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:43.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.027+0000 7fe839341900 10 trimming bilogs for bucket=:dipepu-9[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14]) markers=[,00000000002.17.6,,,,,,,,,], shards=11 2026-03-20T11:46:43.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.027+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff744b970:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.027+0000 7fe839341900 10 trimming bilog shard 1 of :dipepu-9[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14]) at marker 00000000002.17.6 2026-03-20T11:46:43.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.027+0000 7fe839341900 20 run: stack=0x55bff73ec910 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.027+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73ef5d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.027+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73ef5d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.027+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14.1 2026-03-20T11:46:43.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.027+0000 7fe839341900 20 run: stack=0x55bff74261a0 is io blocked 2026-03-20T11:46:43.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73ef5d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73ef5d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe839341900 15 stack 0x55bff74261a0 end 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe839341900 20 run: stack=0x55bff74261a0 is done 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff744b970:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe839341900 20 trim: trimmed bucket instance dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe839341900 15 stack 0x55bff73ec910 end 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe839341900 20 run: stack=0x55bff73ec910 is done 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe839341900 4 starting trim on bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f64070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f64070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe839341900 20 enqueued request req=0x55bff73da3f0 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe839341900 20 req: 0x55bff73da3f0 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe839341900 20 run: stack=0x55bff73ec910 is io blocked 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe7f77fe640 20 dequeued request req=0x55bff73da3f0 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe7f77fe640 20 RGWWQ: empty 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe7f77fe640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 : miss 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe7f77fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe7f77fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe7f77fe640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 info.flags=0x17 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe7f77fe640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 to cache LRU end 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe7f77fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe7f77fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe7f77fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe7f77fe640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-11 : miss 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe7f77fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe7f77fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe7f77fe640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-11 info.flags=0x0 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe7f77fe640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-11 to cache LRU end 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe7f77fe640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-11 : miss 2026-03-20T11:46:43.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.028+0000 7fe7f77fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe7f77fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe7f77fe640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-11 info.flags=0x0 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe7f77fe640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-11 to cache LRU end 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe7f77fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16): adding source pipe: {s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe7f77fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16): adding dest pipe: {s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe7f77fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe7f77fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe7f77fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f64070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f64070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f64070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f64070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f5c9b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 20 run: stack=0x55bff73ec910 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 8fa2138aeb4e250b82eba3fe60763c6f2289a22186a93b1d19cd329c23483125 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:43.026 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114643Z 2026-03-20T11:46:43.027 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:43.027 INFO:teuthology.orchestra.run.vm02.stderr:8fa2138aeb4e250b82eba3fe60763c6f2289a22186a93b1d19cd329c23483125 2026-03-20T11:46:43.027 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:43.027 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:43.027 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:43.027 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:43.027 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 4e1c271f746b17d93af8ba10999849563010de3099f86af8fa5c0d36877c5451 2026-03-20T11:46:43.027 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=4e1c271f746b17d93af8ba10999849563010de3099f86af8fa5c0d36877c5451 2026-03-20T11:46:43.027 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.027 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114643Z 2026-03-20T11:46:43.027 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:43.027 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=28, curl_handle=0x55bff73e8f10 2026-03-20T11:46:43.027 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe839341900 20 run: stack=0x55bff74261a0 is io blocked 2026-03-20T11:46:43.027 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.029+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73e0e60 req_data->id=28, curl_handle=0x55bff73e8f10 2026-03-20T11:46:43.027 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.031+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff74252c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.031+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff74252c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.031+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff74252c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.031+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff74252c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.031+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff7456250:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.031+0000 7fe839341900 15 stack 0x55bff7455ec0 end 2026-03-20T11:46:43.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.031+0000 7fe839341900 20 run: stack=0x55bff7455ec0 is done 2026-03-20T11:46:43.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.031+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff6f70f90:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.031+0000 7fe839341900 20 collect(): s=0x55bff73e0d20 stack=0x55bff7455ec0 is complete 2026-03-20T11:46:43.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.031+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:43.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.031+0000 7fe839341900 10 trimming bilogs for bucket=:dipepu-38[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43]) markers=[,,,,,,,,00000000001.55.6,,], shards=11 2026-03-20T11:46:43.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.031+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff7456250:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.031+0000 7fe839341900 10 trimming bilog shard 8 of :dipepu-38[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43]) at marker 00000000001.55.6 2026-03-20T11:46:43.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.031+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.031+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff73fa180:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.031+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff73fa180:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.031+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43.8 2026-03-20T11:46:43.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.031+0000 7fe839341900 20 run: stack=0x55bff7455ec0 is io blocked 2026-03-20T11:46:43.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff73fa180:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff73fa180:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe839341900 15 stack 0x55bff7455ec0 end 2026-03-20T11:46:43.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe839341900 20 run: stack=0x55bff7455ec0 is done 2026-03-20T11:46:43.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff7456250:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff6f70f90:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe839341900 20 trim: trimmed bucket instance dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:43.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe839341900 15 stack 0x55bff73e0d20 end 2026-03-20T11:46:43.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is done 2026-03-20T11:46:43.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:43.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff6f70f90:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe839341900 4 starting trim on bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:43.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff73fa180:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff73fa180:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe839341900 20 enqueued request req=0x55bff73e2170 2026-03-20T11:46:43.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:43.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe839341900 20 req: 0x55bff73e2170 2026-03-20T11:46:43.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is io blocked 2026-03-20T11:46:43.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe7f67fc640 20 dequeued request req=0x55bff73e2170 2026-03-20T11:46:43.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe7f67fc640 20 RGWWQ: empty 2026-03-20T11:46:43.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe7f67fc640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 : miss 2026-03-20T11:46:43.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe7f67fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe7f67fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:43.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe7f67fc640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 info.flags=0x17 2026-03-20T11:46:43.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe7f67fc640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 to cache LRU end 2026-03-20T11:46:43.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe7f67fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:43.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe7f67fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:43.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe7f67fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:43.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe7f67fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-12 : miss 2026-03-20T11:46:43.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.032+0000 7fe7f67fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe7f67fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe7f67fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-12 info.flags=0x0 2026-03-20T11:46:43.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe7f67fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-12 to cache LRU end 2026-03-20T11:46:43.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe7f67fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-12 : miss 2026-03-20T11:46:43.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe7f67fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe7f67fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe7f67fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-12 info.flags=0x0 2026-03-20T11:46:43.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe7f67fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-12 to cache LRU end 2026-03-20T11:46:43.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe7f67fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17): adding source pipe: {s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe7f67fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17): adding dest pipe: {s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe7f67fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe7f67fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe7f67fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:43.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff73fa180:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff73fa180:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff73fa180:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff73fa180:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff6f70f90:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff7430ad0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff6f70f90:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff7434ea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff7434ea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 401746c5054be763ed9599fe89adf09abcc3024e38c8594b473b028742e36d8a 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114643Z 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:401746c5054be763ed9599fe89adf09abcc3024e38c8594b473b028742e36d8a 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 1ea10d0d8b4e61bd98639ad04d2b70242cc59c1d2b10afe8c363cb035cd98c18 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=1ea10d0d8b4e61bd98639ad04d2b70242cc59c1d2b10afe8c363cb035cd98c18 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114643Z 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=29, curl_handle=0x55bff73f1200 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe839341900 20 run: stack=0x55bff7455ec0 is io blocked 2026-03-20T11:46:43.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.033+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73ed6d0 req_data->id=29, curl_handle=0x55bff73f1200 2026-03-20T11:46:43.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.042+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7455770:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.042+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7455770:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.042+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7455770:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.042+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7455770:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.042+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff744a6a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.042+0000 7fe839341900 15 stack 0x55bff73e9140 end 2026-03-20T11:46:43.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.042+0000 7fe839341900 20 run: stack=0x55bff73e9140 is done 2026-03-20T11:46:43.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.042+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.042+0000 7fe839341900 20 collect(): s=0x55bff72f1ec0 stack=0x55bff73e9140 is complete 2026-03-20T11:46:43.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.042+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:43.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.042+0000 7fe839341900 10 trimming bilogs for bucket=:dipepu-15[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20]) markers=[,,,,,,,,00000000007.15.6,,], shards=11 2026-03-20T11:46:43.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.042+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff7456250:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.042+0000 7fe839341900 10 trimming bilog shard 8 of :dipepu-15[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20]) at marker 00000000007.15.6 2026-03-20T11:46:43.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.042+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.042+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff6f60300:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.042+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff6f60300:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.042+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20.8 2026-03-20T11:46:43.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.042+0000 7fe839341900 20 run: stack=0x55bff73e9140 is io blocked 2026-03-20T11:46:43.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.043+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff6f60300:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.043+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff6f60300:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.043+0000 7fe839341900 15 stack 0x55bff73e9140 end 2026-03-20T11:46:43.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.043+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.043+0000 7fe839341900 20 run: stack=0x55bff73e9140 is done 2026-03-20T11:46:43.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.043+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff7456250:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.043+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.043+0000 7fe839341900 20 trim: trimmed bucket instance dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:43.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.043+0000 7fe839341900 15 stack 0x55bff72f1ec0 end 2026-03-20T11:46:43.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.043+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is done 2026-03-20T11:46:43.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.043+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:43.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.043+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.040 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.043+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.043+0000 7fe839341900 4 starting trim on bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:43.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.043+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.043+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.043+0000 7fe839341900 20 enqueued request req=0x55bff73ed4a0 2026-03-20T11:46:43.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.043+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:43.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.043+0000 7fe839341900 20 req: 0x55bff73ed4a0 2026-03-20T11:46:43.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.043+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is io blocked 2026-03-20T11:46:43.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.043+0000 7fe7f7fff640 20 dequeued request req=0x55bff73ed4a0 2026-03-20T11:46:43.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.043+0000 7fe7f7fff640 20 RGWWQ: empty 2026-03-20T11:46:43.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.043+0000 7fe7f7fff640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 : miss 2026-03-20T11:46:43.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.043+0000 7fe7f7fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe7f7fff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:43.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe7f7fff640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 info.flags=0x17 2026-03-20T11:46:43.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe7f7fff640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 to cache LRU end 2026-03-20T11:46:43.041 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe7f7fff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe7f7fff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff7435c00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe7f7fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe7f7fff640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-40 : miss 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe7f7fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff7435c00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff7435c00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff7435c00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff7430340:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe839341900 15 stack 0x55bff73f32f0 end 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is done 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe839341900 20 collect(): s=0x55bff742c160 stack=0x55bff73f32f0 is complete 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe839341900 10 trimming bilogs for bucket=:dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) markers=[,,,,,00000000002.36.11,,,00000000011.25.6,00000000003.20.6,], shards=11 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7424ca0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe839341900 10 trimming bilog shard 5 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000002.36.11 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe839341900 10 trimming bilog shard 8 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000011.25.6 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe839341900 10 trimming bilog shard 9 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000003.20.6 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe839341900 20 run: stack=0x55bff742c160 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff7430250:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff6f647c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff744cd50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff7430250:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.5 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is io blocked 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff6f647c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.8 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe839341900 20 run: stack=0x55bff73e9140 is io blocked 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff744cd50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.9 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is io blocked 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe7f7fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe7f7fff640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-40 info.flags=0x0 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe7f7fff640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-40 to cache LRU end 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe7f7fff640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-40 : miss 2026-03-20T11:46:43.042 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.044+0000 7fe7f7fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe7f7fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe7f7fff640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-40 info.flags=0x0 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe7f7fff640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-40 to cache LRU end 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe7f7fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1): adding source pipe: {s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe7f7fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1): adding dest pipe: {s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe7f7fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe7f7fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe7f7fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7455770:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff6f5d1a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff6f5d1a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = d387311b935f954f3499d81f4fdf8f3bb03d69354828a92cc0b3dca945a2b217 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114643Z 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:d387311b935f954f3499d81f4fdf8f3bb03d69354828a92cc0b3dca945a2b217 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 932f5161d17d3027830453daa269e3c57b5f0810332c6db40cf6ce19817266b0 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=932f5161d17d3027830453daa269e3c57b5f0810332c6db40cf6ce19817266b0 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114643Z 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=30, curl_handle=0x55bff73e4760 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 run: stack=0x55bff73e14a0 is io blocked 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff6f647c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff6f647c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 15 stack 0x55bff73e9140 end 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 run: stack=0x55bff73e9140 is done 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff7430250:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff7430250:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 15 stack 0x55bff73f32f0 end 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is done 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff744cd50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff744cd50:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 15 stack 0x55bff73e87e0 end 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is done 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7424ca0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7424ca0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7424ca0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 trim: trimmed bucket instance dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 15 stack 0x55bff742c160 end 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 run: stack=0x55bff742c160 is done 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 4 starting trim on bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 enqueued request req=0x55bff73f1540 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 req: 0x55bff73f1540 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe839341900 20 run: stack=0x55bff742c160 is io blocked 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe7f6ffd640 20 dequeued request req=0x55bff73f1540 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe7f6ffd640 20 RGWWQ: empty 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe7f6ffd640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 : miss 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe7e0fe1640 20 link_request req_data=0x55bff6f63e70 req_data->id=30, curl_handle=0x55bff73e4760 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.045+0000 7fe7f6ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe7f6ffd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe7f6ffd640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 info.flags=0x17 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe7f6ffd640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 to cache LRU end 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe7f6ffd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe7f6ffd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe7f6ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe7f6ffd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-18 : miss 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe7f6ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe7f6ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe7f6ffd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-18 info.flags=0x0 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe7f6ffd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-18 to cache LRU end 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe7f6ffd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-18 : miss 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe7f6ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe7f6ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe7f6ffd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-18 info.flags=0x0 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe7f6ffd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-18 to cache LRU end 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe7f6ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23): adding source pipe: {s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe7f6ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23): adding dest pipe: {s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe7f6ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe7f6ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe7f6ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.045 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff7424ca0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe839341900 20 run: stack=0x55bff742c160 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742fb90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742fb90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 173117fe14fb1a8a8916e48141e7ed9ed42ade026134689744277ed789dcbad7 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114643Z 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:173117fe14fb1a8a8916e48141e7ed9ed42ade026134689744277ed789dcbad7 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.046+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = ae0f3c5a886f420c5983baf4087ad2ddac951b68a5458b6de5a6d7bd59648760 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.047+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=ae0f3c5a886f420c5983baf4087ad2ddac951b68a5458b6de5a6d7bd59648760 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.047+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.047+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114643Z 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.047+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.047+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=31, curl_handle=0x55bff73ecfc0 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.047+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is io blocked 2026-03-20T11:46:43.046 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.047+0000 7fe7e0fe1640 20 link_request req_data=0x55bff74368b0 req_data->id=31, curl_handle=0x55bff73ecfc0 2026-03-20T11:46:43.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f5c9b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 15 stack 0x55bff74261a0 end 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 run: stack=0x55bff74261a0 is done 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 collect(): s=0x55bff73ec910 stack=0x55bff74261a0 is complete 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 10 trimming bilogs for bucket=:dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) markers=[00000000014.53.6,00000000008.38.6,00000000008.39.6,00000000008.35.6,00000000012.28.6,00000000010.38.6,00000000008.22.6,00000000010.27.6,00000000004.28.6,00000000008.20.6,00000000008.23.6], shards=11 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f5c9b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 10 trimming bilog shard 0 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000014.53.6 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 10 trimming bilog shard 1 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.38.6 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 10 trimming bilog shard 2 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.39.6 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 10 trimming bilog shard 3 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.35.6 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 10 trimming bilog shard 4 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000012.28.6 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 10 trimming bilog shard 5 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000010.38.6 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 10 trimming bilog shard 6 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.22.6 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 10 trimming bilog shard 7 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000010.27.6 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 10 trimming bilog shard 8 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000004.28.6 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 10 trimming bilog shard 9 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.20.6 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 10 trimming bilog shard 10 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.23.6 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 run: stack=0x55bff73ec910 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73ef5d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff74359d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff73f6b80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff744d980:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff744c120:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 cr:s=0x55bff73f0960:op=0x55bff744c920:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff744f310:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 cr:s=0x55bff73f4020:op=0x55bff744fb10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff7450310:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 cr:s=0x55bff73f6860:op=0x55bff7450b10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 cr:s=0x55bff73f78d0:op=0x55bff7451310:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73ef5d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.0 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 run: stack=0x55bff74261a0 is io blocked 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff74359d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.1 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 run: stack=0x55bff73e9140 is io blocked 2026-03-20T11:46:43.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff73f6b80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.2 2026-03-20T11:46:43.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is io blocked 2026-03-20T11:46:43.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff744d980:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.071+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.3 2026-03-20T11:46:43.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.072+0000 7fe839341900 20 run: stack=0x55bff7432090 is io blocked 2026-03-20T11:46:43.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.072+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff744c120:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.072+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.4 2026-03-20T11:46:43.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.072+0000 7fe839341900 20 run: stack=0x55bff73ef490 is io blocked 2026-03-20T11:46:43.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.072+0000 7fe839341900 20 cr:s=0x55bff73f0960:op=0x55bff744c920:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.072+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.5 2026-03-20T11:46:43.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.072+0000 7fe839341900 20 run: stack=0x55bff73f0960 is io blocked 2026-03-20T11:46:43.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.072+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff744f310:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.072+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.6 2026-03-20T11:46:43.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.072+0000 7fe839341900 20 run: stack=0x55bff6f63190 is io blocked 2026-03-20T11:46:43.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.072+0000 7fe839341900 20 cr:s=0x55bff73f4020:op=0x55bff744fb10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.072+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.7 2026-03-20T11:46:43.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.072+0000 7fe839341900 20 run: stack=0x55bff73f4020 is io blocked 2026-03-20T11:46:43.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.072+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff7450310:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.072+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.8 2026-03-20T11:46:43.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.072+0000 7fe839341900 20 run: stack=0x55bff73e6db0 is io blocked 2026-03-20T11:46:43.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.072+0000 7fe839341900 20 cr:s=0x55bff73f6860:op=0x55bff7450b10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.072+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.9 2026-03-20T11:46:43.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.072+0000 7fe839341900 20 run: stack=0x55bff73f6860 is io blocked 2026-03-20T11:46:43.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.072+0000 7fe839341900 20 cr:s=0x55bff73f78d0:op=0x55bff7451310:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.072+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.10 2026-03-20T11:46:43.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.072+0000 7fe839341900 20 run: stack=0x55bff73f78d0 is io blocked 2026-03-20T11:46:43.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff74359d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff74359d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 15 stack 0x55bff73e9140 end 2026-03-20T11:46:43.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 20 run: stack=0x55bff73e9140 is done 2026-03-20T11:46:43.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f5c9b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 20 run: stack=0x55bff73ec910 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73ef5d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff73ef5d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 15 stack 0x55bff74261a0 end 2026-03-20T11:46:43.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 20 run: stack=0x55bff74261a0 is done 2026-03-20T11:46:43.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff73f6b80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff73f6b80:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 15 stack 0x55bff73f32f0 end 2026-03-20T11:46:43.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is done 2026-03-20T11:46:43.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff744d980:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff744d980:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 15 stack 0x55bff7432090 end 2026-03-20T11:46:43.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 20 run: stack=0x55bff7432090 is done 2026-03-20T11:46:43.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f5c9b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f5c9b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f5c9b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 20 run: stack=0x55bff73ec910 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff744c120:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff744c120:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 15 stack 0x55bff73ef490 end 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 20 run: stack=0x55bff73ef490 is done 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f5c9b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.073+0000 7fe839341900 20 run: stack=0x55bff73ec910 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff744f310:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff744f310:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 15 stack 0x55bff6f63190 end 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 run: stack=0x55bff6f63190 is done 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f5c9b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 run: stack=0x55bff73ec910 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff7450310:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff7450310:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 15 stack 0x55bff73e6db0 end 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 run: stack=0x55bff73e6db0 is done 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 cr:s=0x55bff73f6860:op=0x55bff7450b10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 cr:s=0x55bff73f6860:op=0x55bff7450b10:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 15 stack 0x55bff73f6860 end 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 run: stack=0x55bff73f6860 is done 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f5c9b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f5c9b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 run: stack=0x55bff73ec910 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 cr:s=0x55bff73f78d0:op=0x55bff7451310:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 cr:s=0x55bff73f78d0:op=0x55bff7451310:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 15 stack 0x55bff73f78d0 end 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 run: stack=0x55bff73f78d0 is done 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f5c9b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 run: stack=0x55bff73ec910 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 cr:s=0x55bff73f0960:op=0x55bff744c920:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 cr:s=0x55bff73f0960:op=0x55bff744c920:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 15 stack 0x55bff73f0960 end 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 run: stack=0x55bff73f0960 is done 2026-03-20T11:46:43.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f5c9b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 run: stack=0x55bff73ec910 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 cr:s=0x55bff73f4020:op=0x55bff744fb10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 cr:s=0x55bff73f4020:op=0x55bff744fb10:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 15 stack 0x55bff73f4020 end 2026-03-20T11:46:43.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 run: stack=0x55bff73f4020 is done 2026-03-20T11:46:43.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff6f5c9b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 trim: trimmed bucket instance dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:43.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 15 stack 0x55bff73ec910 end 2026-03-20T11:46:43.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 run: stack=0x55bff73ec910 is done 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 4 starting trim on bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 enqueued request req=0x55bff6f62aa0 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 req: 0x55bff6f62aa0 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe839341900 20 run: stack=0x55bff73e6db0 is io blocked 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe7f5ffb640 20 dequeued request req=0x55bff6f62aa0 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe7f5ffb640 20 RGWWQ: empty 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe7f5ffb640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 : miss 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe7f5ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe7f5ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe7f5ffb640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 info.flags=0x17 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe7f5ffb640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 to cache LRU end 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe7f5ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe7f5ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe7f5ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe7f5ffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-50 : miss 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.074+0000 7fe7f5ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe7f5ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe7f5ffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-50 info.flags=0x0 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe7f5ffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-50 to cache LRU end 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe7f5ffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-50 : miss 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe7f5ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe7f5ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe7f5ffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-50 info.flags=0x0 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe7f5ffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-50 to cache LRU end 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe7f5ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10): adding source pipe: {s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe7f5ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10): adding dest pipe: {s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe7f5ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe7f5ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe7f5ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff6f5c9b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 20 run: stack=0x55bff73e6db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:43.074 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 27fe142093132e49698483627d13b7750c77d8713469d20635913f722a37d16b 2026-03-20T11:46:43.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:43.075 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114643Z 2026-03-20T11:46:43.075 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:43.075 INFO:teuthology.orchestra.run.vm02.stderr:27fe142093132e49698483627d13b7750c77d8713469d20635913f722a37d16b 2026-03-20T11:46:43.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:43.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:43.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:43.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:43.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = a92d172fb9c48eb13b8f3d539548be75e98cf8ad969ecc4e88526ab75f195ebe 2026-03-20T11:46:43.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a92d172fb9c48eb13b8f3d539548be75e98cf8ad969ecc4e88526ab75f195ebe 2026-03-20T11:46:43.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114643Z 2026-03-20T11:46:43.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:43.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=32, curl_handle=0x55bff73e8f10 2026-03-20T11:46:43.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe839341900 20 run: stack=0x55bff6f63190 is io blocked 2026-03-20T11:46:43.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.075+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73e0e60 req_data->id=32, curl_handle=0x55bff73e8f10 2026-03-20T11:46:43.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff7434ea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff7434ea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff7434ea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff7434ea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff7430ad0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 15 stack 0x55bff7455ec0 end 2026-03-20T11:46:43.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 20 run: stack=0x55bff7455ec0 is done 2026-03-20T11:46:43.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff6f70f90:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 20 collect(): s=0x55bff73e0d20 stack=0x55bff7455ec0 is complete 2026-03-20T11:46:43.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:43.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 10 trimming bilogs for bucket=:dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) markers=[00000000014.49.6,00000000008.48.6,00000000008.43.6,00000000008.23.6,00000000012.30.6,00000000010.25.6,00000000008.36.6,00000000010.29.6,00000000004.42.6,00000000008.28.6,00000000008.26.6], shards=11 2026-03-20T11:46:43.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff74230d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 10 trimming bilog shard 0 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000014.49.6 2026-03-20T11:46:43.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 10 trimming bilog shard 1 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.48.6 2026-03-20T11:46:43.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 10 trimming bilog shard 2 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.43.6 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 10 trimming bilog shard 3 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.23.6 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 10 trimming bilog shard 4 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000012.30.6 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 10 trimming bilog shard 5 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000010.25.6 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 10 trimming bilog shard 6 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.36.6 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 10 trimming bilog shard 7 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000010.29.6 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 10 trimming bilog shard 8 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000004.42.6 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 10 trimming bilog shard 9 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.28.6 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 10 trimming bilog shard 10 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.26.6 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff74359d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff7434d90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff73fa180:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff6f5da80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff744c080:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff744d980:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 20 cr:s=0x55bff73f0960:op=0x55bff6f6f3b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff744f170:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 20 cr:s=0x55bff7435840:op=0x55bff7430350:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 20 cr:s=0x55bff73df3b0:op=0x55bff73f6c20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff7450900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff74359d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.0 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 20 run: stack=0x55bff7455ec0 is io blocked 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff7434d90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.076+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.1 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.077+0000 7fe839341900 20 run: stack=0x55bff73ef490 is io blocked 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.077+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff73fa180:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.077+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.2 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.077+0000 7fe839341900 20 run: stack=0x55bff7432090 is io blocked 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.077+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff6f5da80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.077+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.3 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.077+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is io blocked 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.077+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff744c080:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.077+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.4 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.077+0000 7fe839341900 20 run: stack=0x55bff74261a0 is io blocked 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.077+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff744d980:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.077+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.5 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.078+0000 7fe839341900 20 run: stack=0x55bff73e9140 is io blocked 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.078+0000 7fe839341900 20 cr:s=0x55bff73f0960:op=0x55bff6f6f3b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.078+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.6 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.078+0000 7fe839341900 20 run: stack=0x55bff73f0960 is io blocked 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.078+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff744f170:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.078+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.7 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.078+0000 7fe839341900 20 run: stack=0x55bff73ec910 is io blocked 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.078+0000 7fe839341900 20 cr:s=0x55bff7435840:op=0x55bff7430350:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.078+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.8 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 run: stack=0x55bff7435840 is io blocked 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 cr:s=0x55bff73df3b0:op=0x55bff73f6c20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.9 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 run: stack=0x55bff73df3b0 is io blocked 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff7450900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.10 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 run: stack=0x55bff742eb70 is io blocked 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff74359d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff74359d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 15 stack 0x55bff7455ec0 end 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 run: stack=0x55bff7455ec0 is done 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff7434d90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff7434d90:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 15 stack 0x55bff73ef490 end 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 run: stack=0x55bff73ef490 is done 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff6f5da80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff6f5da80:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 15 stack 0x55bff73f32f0 end 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is done 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff73fa180:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff73fa180:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 15 stack 0x55bff7432090 end 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 run: stack=0x55bff7432090 is done 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff74230d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff74230d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff74230d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff74230d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff744d980:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff744d980:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 15 stack 0x55bff73e9140 end 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 run: stack=0x55bff73e9140 is done 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff74230d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.079+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff7450900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff7450900:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 15 stack 0x55bff742eb70 end 2026-03-20T11:46:43.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 run: stack=0x55bff742eb70 is done 2026-03-20T11:46:43.079 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff74230d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff744c080:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff744c080:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 15 stack 0x55bff74261a0 end 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 run: stack=0x55bff74261a0 is done 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 cr:s=0x55bff73f0960:op=0x55bff6f6f3b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 cr:s=0x55bff73f0960:op=0x55bff6f6f3b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 15 stack 0x55bff73f0960 end 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 run: stack=0x55bff73f0960 is done 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff74230d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff74230d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff744f170:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 cr:s=0x55bff73ec910:op=0x55bff744f170:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 15 stack 0x55bff73ec910 end 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 run: stack=0x55bff73ec910 is done 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff74230d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 cr:s=0x55bff73df3b0:op=0x55bff73f6c20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 cr:s=0x55bff73df3b0:op=0x55bff73f6c20:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 15 stack 0x55bff73df3b0 end 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 run: stack=0x55bff73df3b0 is done 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff74230d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 cr:s=0x55bff7435840:op=0x55bff7430350:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 cr:s=0x55bff7435840:op=0x55bff7430350:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 15 stack 0x55bff7435840 end 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 run: stack=0x55bff7435840 is done 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff74230d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 cr:s=0x55bff73e0d20:op=0x55bff6f70f90:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 trim: trimmed bucket instance dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 15 stack 0x55bff73e0d20 end 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 run: stack=0x55bff73e0d20 is done 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f70f30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 4 starting trim on bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 enqueued request req=0x55bff73f4da0 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 req: 0x55bff73f4da0 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe7f57fa640 20 dequeued request req=0x55bff73f4da0 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe7f57fa640 20 RGWWQ: empty 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe7f57fa640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 : miss 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe7f57fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe839341900 20 run: stack=0x55bff74261a0 is io blocked 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe7f57fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe7f57fa640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 info.flags=0x17 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe7f57fa640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 to cache LRU end 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe7f57fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe7f57fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe7f57fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:43.080 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe7f57fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-24 : miss 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.080+0000 7fe7f57fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe7f57fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe7f57fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-24 info.flags=0x0 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe7f57fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-24 to cache LRU end 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe7f57fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-24 : miss 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe7f57fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe7f57fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe7f57fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-24 info.flags=0x0 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe7f57fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-24 to cache LRU end 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe7f57fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29): adding source pipe: {s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe7f57fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29): adding dest pipe: {s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe7f57fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe7f57fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe7f57fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f70f30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff74230d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f70f30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 20 run: stack=0x55bff74261a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff6f64070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff6f64070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 42ea9da4028ef2247191f958edef6eada4535a669fb3644988ab28e6629c38a3 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114643Z 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:42ea9da4028ef2247191f958edef6eada4535a669fb3644988ab28e6629c38a3 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 9c38acebc730e860fcc6fde722bb928cc387dfdc6acdf623f2c281a612d45649 2026-03-20T11:46:43.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=9c38acebc730e860fcc6fde722bb928cc387dfdc6acdf623f2c281a612d45649 2026-03-20T11:46:43.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114643Z 2026-03-20T11:46:43.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:43.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=33, curl_handle=0x55bff73f1200 2026-03-20T11:46:43.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe839341900 20 run: stack=0x55bff742eb70 is io blocked 2026-03-20T11:46:43.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.081+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73ed6d0 req_data->id=33, curl_handle=0x55bff73f1200 2026-03-20T11:46:43.082 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:43.082+0000 7f124e18c640 -1 req 9389411740297493313 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:43.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.088+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff6f5d1a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.088+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff6f5d1a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.088+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff6f5d1a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.088+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff6f5d1a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.088+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7455770:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.088+0000 7fe839341900 15 stack 0x55bff73e14a0 end 2026-03-20T11:46:43.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.088+0000 7fe839341900 20 run: stack=0x55bff73e14a0 is done 2026-03-20T11:46:43.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.088+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.088+0000 7fe839341900 20 collect(): s=0x55bff72f1ec0 stack=0x55bff73e14a0 is complete 2026-03-20T11:46:43.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.088+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:43.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.088+0000 7fe839341900 10 trimming bilogs for bucket=:dipepu-40[4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1]) markers=[,,,,,00000000001.144.6,,,,,], shards=11 2026-03-20T11:46:43.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.088+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74313a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.088+0000 7fe839341900 10 trimming bilog shard 5 of :dipepu-40[4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1]) at marker 00000000001.144.6 2026-03-20T11:46:43.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.088+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.088+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff6f5da80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.088+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff6f5da80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.088+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1.5 2026-03-20T11:46:43.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.088+0000 7fe839341900 20 run: stack=0x55bff73e14a0 is io blocked 2026-03-20T11:46:43.085 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742fb90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742fb90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff6f5da80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff6f5da80:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 15 stack 0x55bff73e14a0 end 2026-03-20T11:46:43.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 run: stack=0x55bff73e14a0 is done 2026-03-20T11:46:43.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742fb90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74313a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742fb90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 trim: trimmed bucket instance dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:43.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 15 stack 0x55bff72f1ec0 end 2026-03-20T11:46:43.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is done 2026-03-20T11:46:43.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff7424ca0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 15 stack 0x55bff73e87e0 end 2026-03-20T11:46:43.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is done 2026-03-20T11:46:43.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:43.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 collect(): s=0x55bff742c160 stack=0x55bff73e87e0 is complete 2026-03-20T11:46:43.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:43.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 10 trimming bilogs for bucket=:dipepu-18[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23]) markers=[,,,,,,,,00000000002.59.6,,], shards=11 2026-03-20T11:46:43.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 4 starting trim on bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:46:43.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7424ca0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.086 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 10 trimming bilog shard 8 of :dipepu-18[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23]) at marker 00000000002.59.6 2026-03-20T11:46:43.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 run: stack=0x55bff742c160 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff6f61450:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 enqueued request req=0x55bff73da3f0 2026-03-20T11:46:43.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:43.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 req: 0x55bff73da3f0 2026-03-20T11:46:43.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe7f4ff9640 20 dequeued request req=0x55bff73da3f0 2026-03-20T11:46:43.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe7f4ff9640 20 RGWWQ: empty 2026-03-20T11:46:43.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe7f4ff9640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 : miss 2026-03-20T11:46:43.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe7f4ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is io blocked 2026-03-20T11:46:43.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff6f61450:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23.8 2026-03-20T11:46:43.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.089+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is io blocked 2026-03-20T11:46:43.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.090+0000 7fe7f4ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:43.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.090+0000 7fe7f4ff9640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 info.flags=0x17 2026-03-20T11:46:43.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.090+0000 7fe7f4ff9640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 to cache LRU end 2026-03-20T11:46:43.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.090+0000 7fe7f4ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:43.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.090+0000 7fe7f4ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:43.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.090+0000 7fe7f4ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:46:43.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.090+0000 7fe7f4ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-56 : miss 2026-03-20T11:46:43.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.090+0000 7fe7f4ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.090+0000 7fe7f4ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.090+0000 7fe7f4ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-56 info.flags=0x0 2026-03-20T11:46:43.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.090+0000 7fe7f4ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-56 to cache LRU end 2026-03-20T11:46:43.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.090+0000 7fe7f4ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-56 : miss 2026-03-20T11:46:43.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.090+0000 7fe7f4ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.090+0000 7fe7f4ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.090+0000 7fe7f4ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-56 info.flags=0x0 2026-03-20T11:46:43.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.090+0000 7fe7f4ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-56 to cache LRU end 2026-03-20T11:46:43.088 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.090+0000 7fe7f4ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16): adding source pipe: {s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.090+0000 7fe7f4ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16): adding dest pipe: {s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.090+0000 7fe7f4ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.090+0000 7fe7f4ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe7f4ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff6f61450:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff6f61450:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 15 stack 0x55bff73e87e0 end 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is done 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff7424ca0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 cr:s=0x55bff742c160:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 trim: trimmed bucket instance dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 15 stack 0x55bff742c160 end 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 run: stack=0x55bff742c160 is done 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 4 starting trim on bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff6f61450:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff6f61450:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 enqueued request req=0x55bff73da3f0 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 req: 0x55bff73da3f0 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe7effff640 20 dequeued request req=0x55bff73da3f0 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe7effff640 20 RGWWQ: empty 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe7effff640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 : miss 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe7effff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is io blocked 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7424ca0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff742fb90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff742fb90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe7effff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe7effff640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 info.flags=0x17 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe7effff640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 to cache LRU end 2026-03-20T11:46:43.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe7effff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe7effff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe7effff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe7effff640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-27 : miss 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe7effff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 7c7a91b39783acb72e1c8072018beb1d3ef0b7317d1da741f184e317bed94db0 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114643Z 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:7c7a91b39783acb72e1c8072018beb1d3ef0b7317d1da741f184e317bed94db0 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 620033b08f3a100515dcc3036733c092982315a583d80ad0cf24bb913e8c7e02 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=620033b08f3a100515dcc3036733c092982315a583d80ad0cf24bb913e8c7e02 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114643Z 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=34, curl_handle=0x55bff73ecfc0 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe839341900 20 run: stack=0x55bff73e14a0 is io blocked 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.091+0000 7fe7e0fe1640 20 link_request req_data=0x55bff74368b0 req_data->id=34, curl_handle=0x55bff73ecfc0 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe7effff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe7effff640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-27 info.flags=0x0 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe7effff640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-27 to cache LRU end 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe7effff640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-27 : miss 2026-03-20T11:46:43.090 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe7effff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.090 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:43.092+0000 7f12f92e2640 -1 req 15074985292174674233 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe7effff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe7effff640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-27 info.flags=0x0 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe7effff640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-27 to cache LRU end 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe7effff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32): adding source pipe: {s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe7effff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32): adding dest pipe: {s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe7effff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe7effff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe7effff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff6f61450:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff6f61450:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff6f61450:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff6f61450:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7456190:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff742ee10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff742ee10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 21ed389f0694a6f1e2233c00c32705daa1abc44304b25b2c8952591885194527 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114643Z 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:21ed389f0694a6f1e2233c00c32705daa1abc44304b25b2c8952591885194527 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:43.091 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:43.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 122cc210cb18d7c24adad9be1079b7b9350530a5458fceef9ee81e47f2ccf97f 2026-03-20T11:46:43.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=122cc210cb18d7c24adad9be1079b7b9350530a5458fceef9ee81e47f2ccf97f 2026-03-20T11:46:43.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114643Z 2026-03-20T11:46:43.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:43.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=35, curl_handle=0x55bff73e4760 2026-03-20T11:46:43.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe839341900 20 run: stack=0x55bff73e9140 is io blocked 2026-03-20T11:46:43.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.092+0000 7fe7e0fe1640 20 link_request req_data=0x55bff6f63e70 req_data->id=35, curl_handle=0x55bff73e4760 2026-03-20T11:46:43.092 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:43.092+0000 7f12871fe640 -1 req 14710374949272888236 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff6f5c9b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe839341900 15 stack 0x55bff6f63190 end 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe839341900 20 run: stack=0x55bff6f63190 is done 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe839341900 20 collect(): s=0x55bff73e6db0 stack=0x55bff6f63190 is complete 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe839341900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe839341900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe839341900 15 stack 0x55bff73e6db0 end 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe839341900 20 stack->operate() returned ret=-22 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe839341900 20 run: stack=0x55bff73e6db0 is done 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe839341900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe839341900 4 starting trim on bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe839341900 20 enqueued request req=0x55bff6f62aa0 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe839341900 20 req: 0x55bff6f62aa0 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe7ef7fe640 20 dequeued request req=0x55bff6f62aa0 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe7ef7fe640 20 RGWWQ: empty 2026-03-20T11:46:43.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe7ef7fe640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 : miss 2026-03-20T11:46:43.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe7ef7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe7ef7fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:43.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe7ef7fe640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 info.flags=0x17 2026-03-20T11:46:43.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe7ef7fe640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 to cache LRU end 2026-03-20T11:46:43.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe7ef7fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:43.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe7ef7fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:43.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe7ef7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:46:43.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe7ef7fe640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-44 : miss 2026-03-20T11:46:43.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe7ef7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe839341900 20 run: stack=0x55bff73e6db0 is io blocked 2026-03-20T11:46:43.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe7ef7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe7ef7fe640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-44 info.flags=0x0 2026-03-20T11:46:43.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe7ef7fe640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-44 to cache LRU end 2026-03-20T11:46:43.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe7ef7fe640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-44 : miss 2026-03-20T11:46:43.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe7ef7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe7ef7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe7ef7fe640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-44 info.flags=0x0 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.116+0000 7fe7ef7fe640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-44 to cache LRU end 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe7ef7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4): adding source pipe: {s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe7ef7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4): adding dest pipe: {s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe7ef7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe7ef7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe7ef7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff6f5c9b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 20 run: stack=0x55bff73e6db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 63d581a38914ff16589b36689f40fab1bce692b5bbdc970ae5e6a2a7bba4de47 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114643Z 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:63d581a38914ff16589b36689f40fab1bce692b5bbdc970ae5e6a2a7bba4de47 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = b65609694ecb5060b6903e251d5e0bdfa570e9819412512d3597a586795ada11 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b65609694ecb5060b6903e251d5e0bdfa570e9819412512d3597a586795ada11 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114643Z 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=36, curl_handle=0x55bff73e8f10 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe839341900 20 run: stack=0x55bff6f63190 is io blocked 2026-03-20T11:46:43.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.117+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73e0e60 req_data->id=36, curl_handle=0x55bff73e8f10 2026-03-20T11:46:43.116 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:43.117+0000 7f12649b9640 -1 req 10057895199070280425 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:43.119 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff6f64070:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff6f64070:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff74230d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe839341900 15 stack 0x55bff742eb70 end 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe839341900 20 run: stack=0x55bff742eb70 is done 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f70f30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe839341900 20 collect(): s=0x55bff74261a0 stack=0x55bff742eb70 is complete 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe839341900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe839341900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f70f30:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe839341900 15 stack 0x55bff74261a0 end 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe839341900 20 stack->operate() returned ret=-22 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe839341900 20 run: stack=0x55bff74261a0 is done 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe839341900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f70f30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe839341900 4 starting trim on bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe839341900 20 enqueued request req=0x55bff73f4da0 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe839341900 20 req: 0x55bff73f4da0 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe7eeffd640 20 dequeued request req=0x55bff73f4da0 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe7eeffd640 20 RGWWQ: empty 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe7eeffd640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 : miss 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe7eeffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe839341900 20 run: stack=0x55bff74261a0 is io blocked 2026-03-20T11:46:43.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe7eeffd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe7eeffd640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 info.flags=0x17 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe7eeffd640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 to cache LRU end 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe7eeffd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe7eeffd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe7eeffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe7eeffd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-17 : miss 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe7eeffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe7eeffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe7eeffd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-17 info.flags=0x0 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe7eeffd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-17 to cache LRU end 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe7eeffd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-17 : miss 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe7eeffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe7eeffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe7eeffd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-17 info.flags=0x0 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe7eeffd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-17 to cache LRU end 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe7eeffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22): adding source pipe: {s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe7eeffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22): adding dest pipe: {s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe7eeffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.123+0000 7fe7eeffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe7eeffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f70f30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff6f64070:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f70f30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 20 run: stack=0x55bff74261a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff7455770:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff7455770:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 7c09e1a9124ea7c155a5d6f616aafc26334c79ebe306952de0556d8664f4bcea 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114643Z 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:7c09e1a9124ea7c155a5d6f616aafc26334c79ebe306952de0556d8664f4bcea 2026-03-20T11:46:43.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:43.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:43.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:43.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:43.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 93d6787bf1f6e56cd5f0db9b73dc063266b44efb954208497c396264543c6516 2026-03-20T11:46:43.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=93d6787bf1f6e56cd5f0db9b73dc063266b44efb954208497c396264543c6516 2026-03-20T11:46:43.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114643Z 2026-03-20T11:46:43.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:43.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=37, curl_handle=0x55bff73f1200 2026-03-20T11:46:43.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe839341900 20 run: stack=0x55bff742eb70 is io blocked 2026-03-20T11:46:43.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.124+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73ed6d0 req_data->id=37, curl_handle=0x55bff73f1200 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff742fb90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff742fb90:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7424ca0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 15 stack 0x55bff73e14a0 end 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 run: stack=0x55bff73e14a0 is done 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff742ee10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff742ee10:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 collect(): s=0x55bff72f1ec0 stack=0x55bff73e14a0 is complete 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 15 stack 0x55bff72f1ec0 end 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 stack->operate() returned ret=-22 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is done 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7456190:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 15 stack 0x55bff73e9140 end 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 run: stack=0x55bff73e9140 is done 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 collect(): s=0x55bff73e87e0 stack=0x55bff73e9140 is complete 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742c300:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 15 stack 0x55bff73e87e0 end 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 stack->operate() returned ret=-22 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is done 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 4 starting trim on bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:43.130 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 4 starting trim on bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 enqueued request req=0x55bff73ed4a0 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 req: 0x55bff73ed4a0 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7edffb640 20 dequeued request req=0x55bff73ed4a0 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7edffb640 20 RGWWQ: empty 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7edffb640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 : miss 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7edffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is io blocked 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff74230d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff74230d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 enqueued request req=0x55bff7431ea0 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 req: 0x55bff7431ea0 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7ee7fc640 20 dequeued request req=0x55bff7431ea0 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7ee7fc640 20 RGWWQ: empty 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7ee7fc640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 : miss 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7ee7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is io blocked 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7edffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7edffb640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 info.flags=0x17 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7edffb640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 to cache LRU end 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7edffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7edffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7edffb640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7edffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7edffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-46 : miss 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7edffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7ee7fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7ee7fc640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 info.flags=0x17 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7ee7fc640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 to cache LRU end 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7ee7fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7ee7fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7ee7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7ee7fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-37 : miss 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7ee7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7edffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7edffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-46 info.flags=0x0 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7edffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-46 to cache LRU end 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7edffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-46 : miss 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.133+0000 7fe7edffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe7edffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe7edffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-46 info.flags=0x0 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe7edffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-46 to cache LRU end 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe7edffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6): adding source pipe: {s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe7edffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6): adding dest pipe: {s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe7edffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe7edffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe7edffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe7ee7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe7ee7fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-37 info.flags=0x0 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe7ee7fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-37 to cache LRU end 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe7ee7fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-37 : miss 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe7ee7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7424ca0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff742ee10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff742ee10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 07513c3cdd929af154cf5f592f7d2f8812f456482d6b88fea870a4139e803a61 2026-03-20T11:46:43.132 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:43.133 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114643Z 2026-03-20T11:46:43.133 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:43.133 INFO:teuthology.orchestra.run.vm02.stderr:07513c3cdd929af154cf5f592f7d2f8812f456482d6b88fea870a4139e803a61 2026-03-20T11:46:43.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:43.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:43.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:43.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:43.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 113eb6858d48718d7fd40c93a3d1e2abe8b405293ff94d81e18a8b5f00007bdb 2026-03-20T11:46:43.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=113eb6858d48718d7fd40c93a3d1e2abe8b405293ff94d81e18a8b5f00007bdb 2026-03-20T11:46:43.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114643Z 2026-03-20T11:46:43.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:43.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=38, curl_handle=0x55bff73e4760 2026-03-20T11:46:43.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 run: stack=0x55bff73e9140 is io blocked 2026-03-20T11:46:43.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe7e0fe1640 20 link_request req_data=0x55bff6f63e70 req_data->id=38, curl_handle=0x55bff73e4760 2026-03-20T11:46:43.133 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:43.134+0000 7f12b8260640 -1 req 11080897350650486456 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:43.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe7ee7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe7ee7fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-37 info.flags=0x0 2026-03-20T11:46:43.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe7ee7fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-37 to cache LRU end 2026-03-20T11:46:43.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe7ee7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42): adding source pipe: {s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe7ee7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42): adding dest pipe: {s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe7ee7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe7ee7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe7ee7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:46:43.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff74230d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff74230d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff74230d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.133 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff74230d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff742fb90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7456190:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7456190:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = eccfc6cfb29ebc76eda9856bc6f735da731eaea6518dee462c87766a11cf424f 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114643Z 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:eccfc6cfb29ebc76eda9856bc6f735da731eaea6518dee462c87766a11cf424f 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 39f4800dd39274fc0abaa83e69ac0cf045f76d6854cb6662ba95f78211b124e3 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=39f4800dd39274fc0abaa83e69ac0cf045f76d6854cb6662ba95f78211b124e3 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114643Z 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=39, curl_handle=0x55bff73ecfc0 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe839341900 20 run: stack=0x55bff73e14a0 is io blocked 2026-03-20T11:46:43.134 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.134+0000 7fe7e0fe1640 20 link_request req_data=0x55bff74368b0 req_data->id=39, curl_handle=0x55bff73ecfc0 2026-03-20T11:46:43.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff6f5c9b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe839341900 15 stack 0x55bff6f63190 end 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe839341900 20 run: stack=0x55bff6f63190 is done 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe839341900 20 collect(): s=0x55bff73e6db0 stack=0x55bff6f63190 is complete 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe839341900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe839341900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe839341900 15 stack 0x55bff73e6db0 end 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe839341900 20 stack->operate() returned ret=-22 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe839341900 20 run: stack=0x55bff73e6db0 is done 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe839341900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe839341900 4 starting trim on bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe839341900 20 enqueued request req=0x55bff6f62aa0 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe839341900 20 req: 0x55bff6f62aa0 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe7ed7fa640 20 dequeued request req=0x55bff6f62aa0 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe7ed7fa640 20 RGWWQ: empty 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe7ed7fa640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 : miss 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe7ed7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe839341900 20 run: stack=0x55bff73e6db0 is io blocked 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe7ed7fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe7ed7fa640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 info.flags=0x17 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe7ed7fa640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 to cache LRU end 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe7ed7fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe7ed7fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe7ed7fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe7ed7fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-1 : miss 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe7ed7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe7ed7fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe7ed7fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-1 info.flags=0x0 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe7ed7fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-1 to cache LRU end 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe7ed7fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-1 : miss 2026-03-20T11:46:43.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe7ed7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe7ed7fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.158+0000 7fe7ed7fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-1 info.flags=0x0 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe7ed7fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-1 to cache LRU end 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe7ed7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1): adding source pipe: {s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe7ed7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1): adding dest pipe: {s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe7ed7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe7ed7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe7ed7fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff6f5c9b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 20 run: stack=0x55bff73e6db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = f76c821ff947cb646ff5d19e6e7807a0b4583d1c65b9c48ebcac4fdd17f72955 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114643Z 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:f76c821ff947cb646ff5d19e6e7807a0b4583d1c65b9c48ebcac4fdd17f72955 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:43.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:43.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 2370a4c85b7edd8503b7b2b79dd2455e6eb07dd0c1ba74b6990d101d58c61eae 2026-03-20T11:46:43.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2370a4c85b7edd8503b7b2b79dd2455e6eb07dd0c1ba74b6990d101d58c61eae 2026-03-20T11:46:43.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114643Z 2026-03-20T11:46:43.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:43.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=40, curl_handle=0x55bff73e8f10 2026-03-20T11:46:43.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe839341900 20 run: stack=0x55bff6f63190 is io blocked 2026-03-20T11:46:43.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.159+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73e0e60 req_data->id=40, curl_handle=0x55bff73e8f10 2026-03-20T11:46:43.157 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:43.159+0000 7f124597b640 -1 req 14211053466326473471 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:43.162 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.166+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff7455770:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.163 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.166+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff7455770:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.163 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.166+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff7455770:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.163 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.166+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff7455770:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.163 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.166+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff6f64070:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.163 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.166+0000 7fe839341900 15 stack 0x55bff742eb70 end 2026-03-20T11:46:43.163 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.166+0000 7fe839341900 20 run: stack=0x55bff742eb70 is done 2026-03-20T11:46:43.163 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.166+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f70f30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.163 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.166+0000 7fe839341900 20 collect(): s=0x55bff74261a0 stack=0x55bff742eb70 is complete 2026-03-20T11:46:43.163 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.166+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:43.163 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.166+0000 7fe839341900 10 trimming bilogs for bucket=:dipepu-17[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22]) markers=[,,,,,,,,00000000005.51.18,,], shards=11 2026-03-20T11:46:43.163 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.166+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f64070:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.163 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.166+0000 7fe839341900 10 trimming bilog shard 8 of :dipepu-17[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22]) at marker 00000000005.51.18 2026-03-20T11:46:43.163 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.166+0000 7fe839341900 20 run: stack=0x55bff74261a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.163 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.166+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff73fa180:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.163 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.166+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff73fa180:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.163 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.166+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22.8 2026-03-20T11:46:43.163 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.166+0000 7fe839341900 20 run: stack=0x55bff742eb70 is io blocked 2026-03-20T11:46:43.164 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff73fa180:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.164 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff73fa180:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.164 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe839341900 15 stack 0x55bff742eb70 end 2026-03-20T11:46:43.164 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.164 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe839341900 20 run: stack=0x55bff742eb70 is done 2026-03-20T11:46:43.164 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f64070:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.164 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f70f30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.164 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe839341900 20 trim: trimmed bucket instance dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:46:43.164 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe839341900 15 stack 0x55bff74261a0 end 2026-03-20T11:46:43.164 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe839341900 20 run: stack=0x55bff74261a0 is done 2026-03-20T11:46:43.164 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:43.164 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.164 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f70f30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.164 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe839341900 4 starting trim on bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:46:43.164 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.164 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.164 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe839341900 20 enqueued request req=0x55bff73f4da0 2026-03-20T11:46:43.164 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:43.164 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe839341900 20 req: 0x55bff73f4da0 2026-03-20T11:46:43.164 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe7ecff9640 20 dequeued request req=0x55bff73f4da0 2026-03-20T11:46:43.164 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe7ecff9640 20 RGWWQ: empty 2026-03-20T11:46:43.164 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe7ecff9640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 : miss 2026-03-20T11:46:43.164 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe7ecff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.164 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe839341900 20 run: stack=0x55bff74261a0 is io blocked 2026-03-20T11:46:43.164 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe7ecff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:46:43.164 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe7ecff9640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 info.flags=0x17 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe7ecff9640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 to cache LRU end 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe7ecff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe7ecff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe7ecff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe7ecff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-7 : miss 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.167+0000 7fe7ecff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe7ecff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe7ecff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-7 info.flags=0x0 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe7ecff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-7 to cache LRU end 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe7ecff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-7 : miss 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe7ecff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe7ecff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe7ecff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-7 info.flags=0x0 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe7ecff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-7 to cache LRU end 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe7ecff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12): adding source pipe: {s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe7ecff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12): adding dest pipe: {s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe7ecff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe7ecff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe7ecff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f70f30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff6f64070:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f70f30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 20 run: stack=0x55bff74261a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff7455770:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff7455770:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.165 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = cc9f2a8d33401d7918c2b72410d4fdec48ec43d8eee9ed31acd0bc156241b40f 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114643Z 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr:cc9f2a8d33401d7918c2b72410d4fdec48ec43d8eee9ed31acd0bc156241b40f 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 0f1c57686d6c7a18694b263250c82b6fcbf6cd8fb197c573936c6a515d7377b1 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=0f1c57686d6c7a18694b263250c82b6fcbf6cd8fb197c573936c6a515d7377b1 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114643Z 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=41, curl_handle=0x55bff73f1200 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe839341900 20 run: stack=0x55bff742eb70 is io blocked 2026-03-20T11:46:43.166 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.168+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73ed6d0 req_data->id=41, curl_handle=0x55bff73f1200 2026-03-20T11:46:43.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff742ee10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.171 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff742ee10:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7424ca0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe839341900 15 stack 0x55bff73e9140 end 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe839341900 20 run: stack=0x55bff73e9140 is done 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe839341900 20 collect(): s=0x55bff72f1ec0 stack=0x55bff73e9140 is complete 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe839341900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe839341900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe839341900 15 stack 0x55bff72f1ec0 end 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe839341900 20 stack->operate() returned ret=-22 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is done 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe839341900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe839341900 4 starting trim on bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe839341900 20 enqueued request req=0x55bff73ed4a0 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe839341900 20 req: 0x55bff73ed4a0 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe7ebff7640 20 dequeued request req=0x55bff73ed4a0 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe7ebff7640 20 RGWWQ: empty 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe7ebff7640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 : miss 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe7ebff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is io blocked 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe7ebff7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe7ebff7640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 info.flags=0x17 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe7ebff7640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 to cache LRU end 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe7ebff7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe7ebff7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe7ebff7640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe7ebff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe7ebff7640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-48 : miss 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe7ebff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe7ebff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe7ebff7640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-48 info.flags=0x0 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe7ebff7640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-48 to cache LRU end 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe7ebff7640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-48 : miss 2026-03-20T11:46:43.172 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.175+0000 7fe7ebff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe7ebff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe7ebff7640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-48 info.flags=0x0 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe7ebff7640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-48 to cache LRU end 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe7ebff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8): adding source pipe: {s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe7ebff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8): adding dest pipe: {s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe7ebff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe7ebff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe7ebff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7456190:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7456190:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7456190:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7456190:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff742fb90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 15 stack 0x55bff73e14a0 end 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 run: stack=0x55bff73e14a0 is done 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 collect(): s=0x55bff73e87e0 stack=0x55bff73e14a0 is complete 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 10 trimming bilogs for bucket=:dipepu-37[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42]) markers=[,,,,,,,,,,00000000001.45.6], shards=11 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff6f61550:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 10 trimming bilog shard 10 of :dipepu-37[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42]) at marker 00000000001.45.6 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7456190:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7435990:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7424ca0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7435990:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42.10 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 run: stack=0x55bff73e9140 is io blocked 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7424ca0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = d86300e8f7523738848f0ea0804130875993d90ca778e3f2864fa63872713600 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114643Z 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:d86300e8f7523738848f0ea0804130875993d90ca778e3f2864fa63872713600 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = b71f02e67687b3a9e59297a44bcee6527ead407aa0aa0ce6900fa1636a535028 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b71f02e67687b3a9e59297a44bcee6527ead407aa0aa0ce6900fa1636a535028 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114643Z 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=42, curl_handle=0x55bff73ecfc0 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe839341900 20 run: stack=0x55bff73e14a0 is io blocked 2026-03-20T11:46:43.174 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.176+0000 7fe7e0fe1640 20 link_request req_data=0x55bff74368b0 req_data->id=42, curl_handle=0x55bff73ecfc0 2026-03-20T11:46:43.175 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:43.176+0000 7f1251192640 -1 req 17134787612166537257 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7435990:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7435990:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe839341900 15 stack 0x55bff73e9140 end 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe839341900 20 run: stack=0x55bff73e9140 is done 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff6f61550:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe839341900 20 trim: trimmed bucket instance dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe839341900 15 stack 0x55bff73e87e0 end 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is done 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe839341900 4 starting trim on bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff73dfd00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff73dfd00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe839341900 20 enqueued request req=0x55bff73da3f0 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe839341900 20 req: 0x55bff73da3f0 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe7ec7f8640 20 dequeued request req=0x55bff73da3f0 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe7ec7f8640 20 RGWWQ: empty 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe7ec7f8640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 : miss 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe7ec7f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is io blocked 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe7ec7f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe7ec7f8640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 info.flags=0x17 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe7ec7f8640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 to cache LRU end 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe7ec7f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe7ec7f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe7ec7f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe7ec7f8640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-59 : miss 2026-03-20T11:46:43.175 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe7ec7f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe7ec7f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe7ec7f8640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-59 info.flags=0x0 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe7ec7f8640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-59 to cache LRU end 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe7ec7f8640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-59 : miss 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe7ec7f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe7ec7f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe7ec7f8640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-59 info.flags=0x0 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe7ec7f8640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-59 to cache LRU end 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe7ec7f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19): adding source pipe: {s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.178+0000 7fe7ec7f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19): adding dest pipe: {s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe7ec7f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe7ec7f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe7ec7f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff73dfd00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff73dfd00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff73dfd00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff73dfd00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff742ee10:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff73dfd00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff73dfd00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 1b1de0c9040ccf59f5bae771055ab22c9543301e6c8615a883125fccdf37aac8 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114643Z 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:1b1de0c9040ccf59f5bae771055ab22c9543301e6c8615a883125fccdf37aac8 2026-03-20T11:46:43.176 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:43.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:43.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:43.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:43.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 1bdd3d6aa2a988ad860edea8ca432f3b0fb7bf92d383dc54e80df0002c21bde9 2026-03-20T11:46:43.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=1bdd3d6aa2a988ad860edea8ca432f3b0fb7bf92d383dc54e80df0002c21bde9 2026-03-20T11:46:43.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114643Z 2026-03-20T11:46:43.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:43.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=43, curl_handle=0x55bff73e4760 2026-03-20T11:46:43.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe839341900 20 run: stack=0x55bff73e9140 is io blocked 2026-03-20T11:46:43.177 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.179+0000 7fe7e0fe1640 20 link_request req_data=0x55bff6f63e70 req_data->id=43, curl_handle=0x55bff73e4760 2026-03-20T11:46:43.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:43.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff6f5c9b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe839341900 15 stack 0x55bff6f63190 end 2026-03-20T11:46:43.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe839341900 20 run: stack=0x55bff6f63190 is done 2026-03-20T11:46:43.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe839341900 20 collect(): s=0x55bff73e6db0 stack=0x55bff6f63190 is complete 2026-03-20T11:46:43.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:43.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe839341900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:43.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe839341900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:43.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:43.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe839341900 15 stack 0x55bff73e6db0 end 2026-03-20T11:46:43.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe839341900 20 stack->operate() returned ret=-22 2026-03-20T11:46:43.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe839341900 20 run: stack=0x55bff73e6db0 is done 2026-03-20T11:46:43.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:43.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe839341900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:43.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe839341900 4 starting trim on bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:46:43.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe839341900 20 enqueued request req=0x55bff6f62aa0 2026-03-20T11:46:43.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:43.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe839341900 20 req: 0x55bff6f62aa0 2026-03-20T11:46:43.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe7eb7f6640 20 dequeued request req=0x55bff6f62aa0 2026-03-20T11:46:43.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe7eb7f6640 20 RGWWQ: empty 2026-03-20T11:46:43.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe7eb7f6640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 : miss 2026-03-20T11:46:43.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe7eb7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe839341900 20 run: stack=0x55bff73e6db0 is io blocked 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe7eb7f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe7eb7f6640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 info.flags=0x17 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe7eb7f6640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 to cache LRU end 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe7eb7f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe7eb7f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe7eb7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe7eb7f6640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-21 : miss 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe7eb7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe7eb7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe7eb7f6640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-21 info.flags=0x0 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe7eb7f6640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-21 to cache LRU end 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe7eb7f6640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-21 : miss 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.200+0000 7fe7eb7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe7eb7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe7eb7f6640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-21 info.flags=0x0 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe7eb7f6640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-21 to cache LRU end 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe7eb7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26): adding source pipe: {s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe7eb7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26): adding dest pipe: {s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe7eb7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe7eb7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe7eb7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff74313a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff6f5c9b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 20 run: stack=0x55bff73e6db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.199 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = b23ac38a2795a9be4c61a03e0d4dbe5c0d7671e2d261d39ff850e90efa717f14 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114643Z 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr:b23ac38a2795a9be4c61a03e0d4dbe5c0d7671e2d261d39ff850e90efa717f14 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = d6d9b38a7ef590dc5bc086c8b48e28b0620c95c0582dddc91dd01ffc5fc3c2d5 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=d6d9b38a7ef590dc5bc086c8b48e28b0620c95c0582dddc91dd01ffc5fc3c2d5 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114643Z 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=44, curl_handle=0x55bff73e8f10 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe839341900 20 run: stack=0x55bff6f63190 is io blocked 2026-03-20T11:46:43.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.201+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73e0e60 req_data->id=44, curl_handle=0x55bff73e8f10 2026-03-20T11:46:43.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff7455770:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff7455770:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff7455770:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff7455770:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff6f64070:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 15 stack 0x55bff742eb70 end 2026-03-20T11:46:43.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 run: stack=0x55bff742eb70 is done 2026-03-20T11:46:43.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f70f30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 collect(): s=0x55bff74261a0 stack=0x55bff742eb70 is complete 2026-03-20T11:46:43.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:43.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 10 trimming bilogs for bucket=:dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) markers=[,00000000002.6.6,,00000000001.11.6,,00000000001.7.6,00000000002.9.6,00000000001.8.6,00000000001.12.6,00000000001.7.6,], shards=11 2026-03-20T11:46:43.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f64070:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 10 trimming bilog shard 1 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000002.6.6 2026-03-20T11:46:43.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 10 trimming bilog shard 3 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.11.6 2026-03-20T11:46:43.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 10 trimming bilog shard 5 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.7.6 2026-03-20T11:46:43.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 10 trimming bilog shard 6 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000002.9.6 2026-03-20T11:46:43.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 10 trimming bilog shard 7 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.8.6 2026-03-20T11:46:43.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 10 trimming bilog shard 8 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.12.6 2026-03-20T11:46:43.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 10 trimming bilog shard 9 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.7.6 2026-03-20T11:46:43.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 run: stack=0x55bff74261a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff73fa180:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff744c5e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff6f6f3b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff744f170:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff7434d40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 cr:s=0x55bff744ab00:op=0x55bff7430330:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 cr:s=0x55bff744cde0:op=0x55bff744fb90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff73fa180:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.1 2026-03-20T11:46:43.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 run: stack=0x55bff742eb70 is io blocked 2026-03-20T11:46:43.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff744c5e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.3 2026-03-20T11:46:43.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is io blocked 2026-03-20T11:46:43.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff6f6f3b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.5 2026-03-20T11:46:43.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 run: stack=0x55bff7432090 is io blocked 2026-03-20T11:46:43.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff744f170:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.6 2026-03-20T11:46:43.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 run: stack=0x55bff73ef490 is io blocked 2026-03-20T11:46:43.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff7434d40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.7 2026-03-20T11:46:43.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 run: stack=0x55bff7455ec0 is io blocked 2026-03-20T11:46:43.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 cr:s=0x55bff744ab00:op=0x55bff7430330:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.8 2026-03-20T11:46:43.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 run: stack=0x55bff744ab00 is io blocked 2026-03-20T11:46:43.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 cr:s=0x55bff744cde0:op=0x55bff744fb90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.9 2026-03-20T11:46:43.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.210+0000 7fe839341900 20 run: stack=0x55bff744cde0 is io blocked 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.211+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff73fa180:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.211+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff73fa180:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.211+0000 7fe839341900 15 stack 0x55bff742eb70 end 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.211+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.211+0000 7fe839341900 20 run: stack=0x55bff742eb70 is done 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.211+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f64070:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.211+0000 7fe839341900 20 run: stack=0x55bff74261a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.211+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff744c5e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.211+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff744c5e0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.211+0000 7fe839341900 15 stack 0x55bff73f32f0 end 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.211+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.211+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is done 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.211+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f64070:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.211+0000 7fe839341900 20 run: stack=0x55bff74261a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.211+0000 7fe839341900 20 cr:s=0x55bff744ab00:op=0x55bff7430330:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.211+0000 7fe839341900 20 cr:s=0x55bff744ab00:op=0x55bff7430330:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.211+0000 7fe839341900 15 stack 0x55bff744ab00 end 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.211+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.211+0000 7fe839341900 20 run: stack=0x55bff744ab00 is done 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.211+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f64070:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.211+0000 7fe839341900 20 run: stack=0x55bff74261a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff6f6f3b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff6f6f3b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 15 stack 0x55bff7432090 end 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 run: stack=0x55bff7432090 is done 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f64070:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 run: stack=0x55bff74261a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 cr:s=0x55bff744cde0:op=0x55bff744fb90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 cr:s=0x55bff744cde0:op=0x55bff744fb90:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 15 stack 0x55bff744cde0 end 2026-03-20T11:46:43.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 run: stack=0x55bff744cde0 is done 2026-03-20T11:46:43.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f64070:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 run: stack=0x55bff74261a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff7434d40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 cr:s=0x55bff7455ec0:op=0x55bff7434d40:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 15 stack 0x55bff7455ec0 end 2026-03-20T11:46:43.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 run: stack=0x55bff7455ec0 is done 2026-03-20T11:46:43.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f64070:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 run: stack=0x55bff74261a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff744f170:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 cr:s=0x55bff73ef490:op=0x55bff744f170:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 15 stack 0x55bff73ef490 end 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 run: stack=0x55bff73ef490 is done 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f64070:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 cr:s=0x55bff74261a0:op=0x55bff6f70f30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 trim: trimmed bucket instance dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 15 stack 0x55bff74261a0 end 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 run: stack=0x55bff74261a0 is done 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff6f70f30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 4 starting trim on bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff73f7290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff73f7290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 enqueued request req=0x55bff73e4da0 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 req: 0x55bff73e4da0 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe839341900 20 run: stack=0x55bff7432090 is io blocked 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe7eaff5640 20 dequeued request req=0x55bff73e4da0 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe7eaff5640 20 RGWWQ: empty 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe7eaff5640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 : miss 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.212+0000 7fe7eaff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.213+0000 7fe7eaff5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.213+0000 7fe7eaff5640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 info.flags=0x17 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.213+0000 7fe7eaff5640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 to cache LRU end 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.213+0000 7fe7eaff5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.213+0000 7fe7eaff5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.213+0000 7fe7eaff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.213+0000 7fe7eaff5640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-41 : miss 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.213+0000 7fe7eaff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.213+0000 7fe7eaff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.213+0000 7fe7eaff5640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-41 info.flags=0x0 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.213+0000 7fe7eaff5640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-41 to cache LRU end 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.213+0000 7fe7eaff5640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-41 : miss 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.213+0000 7fe7eaff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.213+0000 7fe7eaff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.213+0000 7fe7eaff5640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-41 info.flags=0x0 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.213+0000 7fe7eaff5640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-41 to cache LRU end 2026-03-20T11:46:43.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.213+0000 7fe7eaff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1): adding source pipe: {s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.213+0000 7fe7eaff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1): adding dest pipe: {s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe7eaff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe7eaff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe7eaff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff73f7290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff73f7290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff73f7290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff73f7290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff6f70f30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 20 cr:s=0x55bff744ab00:op=0x55bff6f64070:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff6f70f30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 20 run: stack=0x55bff7432090 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 20 cr:s=0x55bff744ab00:op=0x55bff7455770:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 20 cr:s=0x55bff744ab00:op=0x55bff7455770:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = d9e7f25a787b22bae81eef379fcc565c0600eeaf2a5aafec22ac88b7a5f625db 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114643Z 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:d9e7f25a787b22bae81eef379fcc565c0600eeaf2a5aafec22ac88b7a5f625db 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 55656c7895400f1dbdec5a28b01d63470b47fe2b09326524f0b4baf0b6fe9b25 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=55656c7895400f1dbdec5a28b01d63470b47fe2b09326524f0b4baf0b6fe9b25 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114643Z 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=45, curl_handle=0x55bff73f1200 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe839341900 20 run: stack=0x55bff744ab00 is io blocked 2026-03-20T11:46:43.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.214+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73ed6d0 req_data->id=45, curl_handle=0x55bff73f1200 2026-03-20T11:46:43.213 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:43.215+0000 7f1222134640 -1 req 17108484899914053154 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:43.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7424ca0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7424ca0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7456190:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe839341900 15 stack 0x55bff73e14a0 end 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe839341900 20 run: stack=0x55bff73e14a0 is done 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe839341900 20 collect(): s=0x55bff72f1ec0 stack=0x55bff73e14a0 is complete 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe839341900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe839341900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe839341900 15 stack 0x55bff72f1ec0 end 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe839341900 20 stack->operate() returned ret=-22 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is done 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe839341900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe839341900 4 starting trim on bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73f7290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73f7290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe839341900 20 enqueued request req=0x55bff7431ea0 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe839341900 20 req: 0x55bff7431ea0 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is io blocked 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe7ea7f4640 20 dequeued request req=0x55bff7431ea0 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe7ea7f4640 20 RGWWQ: empty 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe7ea7f4640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 : miss 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe7ea7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe7ea7f4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe7ea7f4640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 info.flags=0x17 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe7ea7f4640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 to cache LRU end 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe7ea7f4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe7ea7f4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe7ea7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe7ea7f4640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-49 : miss 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe7ea7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe7ea7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.217+0000 7fe7ea7f4640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-49 info.flags=0x0 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe7ea7f4640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-49 to cache LRU end 2026-03-20T11:46:43.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe7ea7f4640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-49 : miss 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe7ea7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe7ea7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe7ea7f4640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-49 info.flags=0x0 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe7ea7f4640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-49 to cache LRU end 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe7ea7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9): adding source pipe: {s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe7ea7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9): adding dest pipe: {s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe7ea7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe7ea7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe7ea7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73f7290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73f7290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73f7290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff73f7290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7456190:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7424ca0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7424ca0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 4f7f2cb90df9a842b5bd84b7f5d2a1ee5fea57516d413687a0c4274badfcd298 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114643Z 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:4f7f2cb90df9a842b5bd84b7f5d2a1ee5fea57516d413687a0c4274badfcd298 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = a19a06679775f0eabb6ede2aece91599b9dfd01b49ca5b3e3d18da8c96a3175d 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a19a06679775f0eabb6ede2aece91599b9dfd01b49ca5b3e3d18da8c96a3175d 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114643Z 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:43.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=46, curl_handle=0x55bff73ecfc0 2026-03-20T11:46:43.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe839341900 20 run: stack=0x55bff73e14a0 is io blocked 2026-03-20T11:46:43.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.218+0000 7fe7e0fe1640 20 link_request req_data=0x55bff74368b0 req_data->id=46, curl_handle=0x55bff73ecfc0 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff73dfd00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff73dfd00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff73dfd00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff73dfd00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff742ee10:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 15 stack 0x55bff73e9140 end 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 20 run: stack=0x55bff73e9140 is done 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 20 collect(): s=0x55bff73e87e0 stack=0x55bff73e9140 is complete 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 10 trimming bilogs for bucket=:dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) markers=[,00000000001.65.6,,,,00000000001.44.6,,,00000000001.58.6,,00000000001.63.6], shards=11 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742ee10:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 10 trimming bilog shard 1 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.65.6 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 10 trimming bilog shard 5 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.44.6 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 10 trimming bilog shard 8 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.58.6 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 10 trimming bilog shard 10 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.63.6 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7434d40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff7453540:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff7435990:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 20 cr:s=0x55bff6f61880:op=0x55bff7450180:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7434d40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.1 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 20 run: stack=0x55bff73e9140 is io blocked 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff7453540:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.5 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is io blocked 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff7435990:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.8 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 20 run: stack=0x55bff742eb70 is io blocked 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 20 cr:s=0x55bff6f61880:op=0x55bff7450180:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.10 2026-03-20T11:46:43.218 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.221+0000 7fe839341900 20 run: stack=0x55bff6f61880 is io blocked 2026-03-20T11:46:43.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.222+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff7453540:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.222+0000 7fe839341900 20 cr:s=0x55bff73f32f0:op=0x55bff7453540:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.222+0000 7fe839341900 15 stack 0x55bff73f32f0 end 2026-03-20T11:46:43.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.222+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.222+0000 7fe839341900 20 run: stack=0x55bff73f32f0 is done 2026-03-20T11:46:43.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.222+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742ee10:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.222+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.222+0000 7fe839341900 20 cr:s=0x55bff6f61880:op=0x55bff7450180:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.222+0000 7fe839341900 20 cr:s=0x55bff6f61880:op=0x55bff7450180:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.222+0000 7fe839341900 15 stack 0x55bff6f61880 end 2026-03-20T11:46:43.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.222+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.222+0000 7fe839341900 20 run: stack=0x55bff6f61880 is done 2026-03-20T11:46:43.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.222+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742ee10:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.222+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.219 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7434d40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe839341900 20 cr:s=0x55bff73e9140:op=0x55bff7434d40:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe839341900 15 stack 0x55bff73e9140 end 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe839341900 20 run: stack=0x55bff73e9140 is done 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742ee10:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff7435990:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff7435990:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe839341900 15 stack 0x55bff742eb70 end 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe839341900 20 run: stack=0x55bff742eb70 is done 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742ee10:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe839341900 20 trim: trimmed bucket instance dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe839341900 15 stack 0x55bff73e87e0 end 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is done 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe839341900 4 starting trim on bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff73f7290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff73f7290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe839341900 20 enqueued request req=0x55bff73e2170 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe839341900 20 req: 0x55bff73e2170 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is io blocked 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe7e9ff3640 20 dequeued request req=0x55bff73e2170 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe7e9ff3640 20 RGWWQ: empty 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe7e9ff3640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 : miss 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe7e9ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe7e9ff3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe7e9ff3640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 info.flags=0x17 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe7e9ff3640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 to cache LRU end 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe7e9ff3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe7e9ff3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe7e9ff3640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe7e9ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe7e9ff3640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-45 : miss 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe7e9ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe7e9ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe7e9ff3640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-45 info.flags=0x0 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe7e9ff3640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-45 to cache LRU end 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe7e9ff3640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-45 : miss 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.223+0000 7fe7e9ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe7e9ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.220 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe7e9ff3640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-45 info.flags=0x0 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe7e9ff3640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-45 to cache LRU end 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe7e9ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5): adding source pipe: {s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe7e9ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5): adding dest pipe: {s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe7e9ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe7e9ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe7e9ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff73f7290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff73f7290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff73f7290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff73f7290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff742ee10:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff73dfd00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff73dfd00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 9b412c8404555e2d77e87018812289d0142f71e12c22fc5ebf747937b5fd0074 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114643Z 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:9b412c8404555e2d77e87018812289d0142f71e12c22fc5ebf747937b5fd0074 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = c53f673c03693495e1a9dbde7643452609dc6b5d6d26e1218c835a820c8f666d 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c53f673c03693495e1a9dbde7643452609dc6b5d6d26e1218c835a820c8f666d 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114643Z 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=47, curl_handle=0x55bff73e4760 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe839341900 20 run: stack=0x55bff742eb70 is io blocked 2026-03-20T11:46:43.221 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.224+0000 7fe7e0fe1640 20 link_request req_data=0x55bff6f63e70 req_data->id=47, curl_handle=0x55bff73e4760 2026-03-20T11:46:43.222 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:43.224+0000 7f12c2274640 -1 req 8548010497058303744 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:43.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.243+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.243+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.243+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.243+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.243+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff6f5c9b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.243+0000 7fe839341900 15 stack 0x55bff6f63190 end 2026-03-20T11:46:43.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.243+0000 7fe839341900 20 run: stack=0x55bff6f63190 is done 2026-03-20T11:46:43.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.243+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.243+0000 7fe839341900 20 collect(): s=0x55bff73e6db0 stack=0x55bff6f63190 is complete 2026-03-20T11:46:43.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.243+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:43.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.243+0000 7fe839341900 10 trimming bilogs for bucket=:dipepu-21[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26]) markers=[,,,,,,,,00000000005.84.14,,], shards=11 2026-03-20T11:46:43.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.243+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff7450470:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.243+0000 7fe839341900 10 trimming bilog shard 8 of :dipepu-21[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26]) at marker 00000000005.84.14 2026-03-20T11:46:43.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.243+0000 7fe839341900 20 run: stack=0x55bff73e6db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.243+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff7434d40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.243+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff7434d40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.243+0000 7fe839341900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26.8 2026-03-20T11:46:43.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.243+0000 7fe839341900 20 run: stack=0x55bff6f63190 is io blocked 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff7434d40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff7434d40:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe839341900 15 stack 0x55bff6f63190 end 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe839341900 20 stack->operate() returned ret=-61 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe839341900 20 run: stack=0x55bff6f63190 is done 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff7450470:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe839341900 20 trim: trimmed bucket instance dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe839341900 15 stack 0x55bff73e6db0 end 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe839341900 20 run: stack=0x55bff73e6db0 is done 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe839341900 4 starting trim on bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73f7290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73f7290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe839341900 20 enqueued request req=0x55bff6f62aa0 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe839341900 20 req: 0x55bff6f62aa0 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe839341900 20 run: stack=0x55bff73e6db0 is io blocked 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe7e8ff1640 20 dequeued request req=0x55bff6f62aa0 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe7e8ff1640 20 RGWWQ: empty 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe7e8ff1640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 : miss 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe7e8ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe7e8ff1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe7e8ff1640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 info.flags=0x17 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe7e8ff1640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 to cache LRU end 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe7e8ff1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe7e8ff1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe7e8ff1640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:46:43.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe7e8ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe7e8ff1640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-33 : miss 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.244+0000 7fe7e8ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe7e8ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe7e8ff1640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-33 info.flags=0x0 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe7e8ff1640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-33 to cache LRU end 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe7e8ff1640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-33 : miss 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe7e8ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe7e8ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe7e8ff1640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-33 info.flags=0x0 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe7e8ff1640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-33 to cache LRU end 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe7e8ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38): adding source pipe: {s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe7e8ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38): adding dest pipe: {s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe7e8ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe7e8ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe7e8ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73f7290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73f7290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73f7290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73f7290:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff6f5c9b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 20 run: stack=0x55bff73e6db0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:43.243 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:43.243 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:43.243 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.243 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.243 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.243 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.243 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.243 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:43.243 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = bff27ecee6967c30fad10aa82f8ac7d6f1d9a5c7ec790817d44cd1b89ad73601 2026-03-20T11:46:43.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:43.243 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114643Z 2026-03-20T11:46:43.243 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:43.243 INFO:teuthology.orchestra.run.vm02.stderr:bff27ecee6967c30fad10aa82f8ac7d6f1d9a5c7ec790817d44cd1b89ad73601 2026-03-20T11:46:43.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:43.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:43.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:43.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:43.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 79413ea5b3c7a330653e6f64be50d126424385a0ed5961a0582db98e4ae0d3e0 2026-03-20T11:46:43.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=79413ea5b3c7a330653e6f64be50d126424385a0ed5961a0582db98e4ae0d3e0 2026-03-20T11:46:43.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114643Z 2026-03-20T11:46:43.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:43.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=48, curl_handle=0x55bff73e8f10 2026-03-20T11:46:43.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe839341900 20 run: stack=0x55bff6f63190 is io blocked 2026-03-20T11:46:43.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.245+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73e0e60 req_data->id=48, curl_handle=0x55bff73e8f10 2026-03-20T11:46:43.243 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:43.245+0000 7f12d8aa1640 -1 req 2086608341925550349 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:43.252 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe839341900 20 cr:s=0x55bff744ab00:op=0x55bff7455770:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe839341900 20 cr:s=0x55bff744ab00:op=0x55bff7455770:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe839341900 20 cr:s=0x55bff744ab00:op=0x55bff6f64070:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe839341900 15 stack 0x55bff744ab00 end 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe839341900 20 run: stack=0x55bff744ab00 is done 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff6f70f30:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe839341900 20 collect(): s=0x55bff7432090 stack=0x55bff744ab00 is complete 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe839341900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe839341900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff6f70f30:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe839341900 15 stack 0x55bff7432090 end 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe839341900 20 stack->operate() returned ret=-22 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe839341900 20 run: stack=0x55bff7432090 is done 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe839341900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7454250:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe839341900 4 starting trim on bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe839341900 20 enqueued request req=0x55bff73e4da0 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe839341900 20 RGWWQ: 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe839341900 20 req: 0x55bff73e4da0 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe839341900 20 run: stack=0x55bff7432090 is io blocked 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe7e97f2640 20 dequeued request req=0x55bff73e4da0 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe7e97f2640 20 RGWWQ: empty 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe7e97f2640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 : miss 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe7e97f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe7e97f2640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe7e97f2640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 info.flags=0x17 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe7e97f2640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 to cache LRU end 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe7e97f2640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe7e97f2640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe7e97f2640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe7e97f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe7e97f2640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-47 : miss 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe7e97f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe7e97f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe7e97f2640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-47 info.flags=0x0 2026-03-20T11:46:43.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe7e97f2640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-47 to cache LRU end 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe7e97f2640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-47 : miss 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.256+0000 7fe7e97f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe7e97f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe7e97f2640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-47 info.flags=0x0 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe7e97f2640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-47 to cache LRU end 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe7e97f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7): adding source pipe: {s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe7e97f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7): adding dest pipe: {s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe7e97f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe7e97f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe7e97f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7455770:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7454250:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 20 cr:s=0x55bff744ab00:op=0x55bff6f64070:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7454250:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 20 run: stack=0x55bff7432090 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 20 cr:s=0x55bff744ab00:op=0x55bff7423520:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 20 cr:s=0x55bff744ab00:op=0x55bff7423520:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:43 +0000 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114643Z 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = da5fb29a61a8420a4df7c126635c8dcf28da24c3a8ee3f733fc40052b5adbdae 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114643Z 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:da5fb29a61a8420a4df7c126635c8dcf28da24c3a8ee3f733fc40052b5adbdae 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = b6e11022d7acd3da700b39b95be7cef40fdc9a297f95cb8d53238c43ec6aaf88 2026-03-20T11:46:43.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b6e11022d7acd3da700b39b95be7cef40fdc9a297f95cb8d53238c43ec6aaf88 2026-03-20T11:46:43.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:43.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114643Z 2026-03-20T11:46:43.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:43.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 20 register_request mgr=0x7ffd73811530 req_data->id=49, curl_handle=0x55bff73f1200 2026-03-20T11:46:43.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe839341900 20 run: stack=0x55bff744ab00 is io blocked 2026-03-20T11:46:43.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.257+0000 7fe7e0fe1640 20 link_request req_data=0x55bff73ed6d0 req_data->id=49, curl_handle=0x55bff73f1200 2026-03-20T11:46:43.255 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:43.257+0000 7f126a1c4640 -1 req 13108222166584467078 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:43.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.259+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7424ca0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.259+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7424ca0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.259+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7424ca0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.259+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7424ca0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.259+0000 7fe839341900 20 cr:s=0x55bff73e14a0:op=0x55bff7456190:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.259+0000 7fe839341900 15 stack 0x55bff73e14a0 end 2026-03-20T11:46:43.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.259+0000 7fe839341900 20 run: stack=0x55bff73e14a0 is done 2026-03-20T11:46:43.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.259+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.259+0000 7fe839341900 20 collect(): s=0x55bff72f1ec0 stack=0x55bff73e14a0 is complete 2026-03-20T11:46:43.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.259+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:43.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.259+0000 7fe839341900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:43.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.259+0000 7fe839341900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:43.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.259+0000 7fe839341900 20 cr:s=0x55bff72f1ec0:op=0x55bff6f5f030:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:43.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.259+0000 7fe839341900 15 stack 0x55bff72f1ec0 end 2026-03-20T11:46:43.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.259+0000 7fe839341900 20 stack->operate() returned ret=-22 2026-03-20T11:46:43.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.259+0000 7fe839341900 20 run: stack=0x55bff72f1ec0 is done 2026-03-20T11:46:43.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.259+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:43.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.259+0000 7fe839341900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:43.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.259+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.261 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.265+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff73dfd00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.261 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.265+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff73dfd00:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:43.261 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.265+0000 7fe839341900 20 cr:s=0x55bff742eb70:op=0x55bff742ee10:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.261 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.265+0000 7fe839341900 15 stack 0x55bff742eb70 end 2026-03-20T11:46:43.261 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.265+0000 7fe839341900 20 run: stack=0x55bff742eb70 is done 2026-03-20T11:46:43.261 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.265+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742c300:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.262 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.265+0000 7fe839341900 20 collect(): s=0x55bff73e87e0 stack=0x55bff742eb70 is complete 2026-03-20T11:46:43.262 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.265+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:43.262 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.265+0000 7fe839341900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:43.262 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.265+0000 7fe839341900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:43.262 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.265+0000 7fe839341900 20 cr:s=0x55bff73e87e0:op=0x55bff742c300:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:43.262 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.265+0000 7fe839341900 15 stack 0x55bff73e87e0 end 2026-03-20T11:46:43.262 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.265+0000 7fe839341900 20 stack->operate() returned ret=-22 2026-03-20T11:46:43.262 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.265+0000 7fe839341900 20 run: stack=0x55bff73e87e0 is done 2026-03-20T11:46:43.262 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.265+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:43.262 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.265+0000 7fe839341900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:43.262 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.265+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.286+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.286+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff7429b00:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:43.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.286+0000 7fe839341900 20 cr:s=0x55bff6f63190:op=0x55bff6f5c9b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.286+0000 7fe839341900 15 stack 0x55bff6f63190 end 2026-03-20T11:46:43.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.286+0000 7fe839341900 20 run: stack=0x55bff6f63190 is done 2026-03-20T11:46:43.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.286+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.286+0000 7fe839341900 20 collect(): s=0x55bff73e6db0 stack=0x55bff6f63190 is complete 2026-03-20T11:46:43.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.286+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:43.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.286+0000 7fe839341900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:43.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.286+0000 7fe839341900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:43.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.286+0000 7fe839341900 20 cr:s=0x55bff73e6db0:op=0x55bff73dc7a0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:43.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.286+0000 7fe839341900 15 stack 0x55bff73e6db0 end 2026-03-20T11:46:43.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.286+0000 7fe839341900 20 stack->operate() returned ret=-22 2026-03-20T11:46:43.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.286+0000 7fe839341900 20 run: stack=0x55bff73e6db0 is done 2026-03-20T11:46:43.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.286+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:43.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.286+0000 7fe839341900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:43.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.286+0000 7fe839341900 20 run: stack=0x55bff73d7750 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:43.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.298+0000 7fe839341900 20 cr:s=0x55bff744ab00:op=0x55bff7423520:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:43.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.298+0000 7fe839341900 20 cr:s=0x55bff744ab00:op=0x55bff7423520:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:43.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.298+0000 7fe839341900 20 cr:s=0x55bff744ab00:op=0x55bff6f64070:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:43.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.298+0000 7fe839341900 15 stack 0x55bff744ab00 end 2026-03-20T11:46:43.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.298+0000 7fe839341900 20 run: stack=0x55bff744ab00 is done 2026-03-20T11:46:43.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.298+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7454250:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:43.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.298+0000 7fe839341900 20 collect(): s=0x55bff7432090 stack=0x55bff744ab00 is complete 2026-03-20T11:46:43.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.298+0000 7fe839341900 10 min_generation is 0 2026-03-20T11:46:43.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.298+0000 7fe839341900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:43.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.298+0000 7fe839341900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:43.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.298+0000 7fe839341900 20 cr:s=0x55bff7432090:op=0x55bff7454250:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:43.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.298+0000 7fe839341900 15 stack 0x55bff7432090 end 2026-03-20T11:46:43.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.298+0000 7fe839341900 20 stack->operate() returned ret=-22 2026-03-20T11:46:43.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.298+0000 7fe839341900 20 run: stack=0x55bff7432090 is done 2026-03-20T11:46:43.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.298+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:43.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.298+0000 7fe839341900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:43.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.298+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73db310:27BucketTrimInstanceCollectCR: operate() returned r=-22 2026-03-20T11:46:43.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.298+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73d7050:12BucketTrimCR: operate() 2026-03-20T11:46:43.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.298+0000 7fe839341900 20 writing bucket trim marker=16:a04872b5:root::dipepu-33:head 2026-03-20T11:46:43.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.298+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff7455770:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:43.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.298+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff7455770:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:43.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.298+0000 7fe839341900 20 run: stack=0x55bff73d7750 is io blocked 2026-03-20T11:46:43.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.300+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff7455770:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:43.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.300+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff7455770:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:43.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.300+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff7455770:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:43.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.300+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff7455770:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:43.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.300+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73d7050:12BucketTrimCR: operate() 2026-03-20T11:46:43.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.300+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff6f61180:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:43.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.300+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff6f61180:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:43.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.300+0000 7fe839341900 20 run: stack=0x55bff73d7750 is io blocked 2026-03-20T11:46:43.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.300+0000 7fe7fffff640 20 trim: bucket trim completed 2026-03-20T11:46:43.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.300+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff6f61180:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:43.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.300+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff6f61180:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:43.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.300+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff6f61180:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:43.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.300+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff6f61180:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:43.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.300+0000 7fe839341900 20 cr:s=0x55bff73d7750:op=0x55bff73d7050:12BucketTrimCR: operate() 2026-03-20T11:46:43.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.300+0000 7fe839341900 4 bucket index log processing completed in 0.747820556s 2026-03-20T11:46:43.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.301+0000 7fe839341900 15 stack 0x55bff73d7750 end 2026-03-20T11:46:43.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.301+0000 7fe839341900 20 run: stack=0x55bff73d7750 is done 2026-03-20T11:46:43.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.306+0000 7fe839341900 20 remove_watcher() i=1 2026-03-20T11:46:43.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.306+0000 7fe839341900 2 removed watcher, disabling cache 2026-03-20T11:46:43.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.307+0000 7fe839341900 20 remove_watcher() i=3 2026-03-20T11:46:43.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.307+0000 7fe839341900 20 remove_watcher() i=6 2026-03-20T11:46:43.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.307+0000 7fe839341900 20 remove_watcher() i=2 2026-03-20T11:46:43.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.307+0000 7fe839341900 20 remove_watcher() i=0 2026-03-20T11:46:43.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.307+0000 7fe839341900 20 remove_watcher() i=7 2026-03-20T11:46:43.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.307+0000 7fe839341900 20 remove_watcher() i=4 2026-03-20T11:46:43.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:43.307+0000 7fe839341900 20 remove_watcher() i=5 2026-03-20T11:46:44.397 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:44.398+0000 7fcfb69bb640 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2026-03-20T11:46:44.397 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:44.398+0000 7fcfaf9ad640 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2026-03-20T11:46:44.397 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:44.398+0000 7fcfb99c1640 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2026-03-20T11:46:44.397 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:44.399+0000 7fcfaf1ac640 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2026-03-20T11:46:44.397 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:44.399+0000 7fcfb91c0640 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2026-03-20T11:46:44.397 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:44.399+0000 7fcfba1c2640 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2026-03-20T11:46:44.397 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:44.399+0000 7fcfaf1ac640 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2026-03-20T11:46:44.397 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:44.399+0000 7fcfb71bc640 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2026-03-20T11:46:48.310 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '50', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:46:48.310 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '50', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:46:48.310 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bilog autotrim --rgw-sync-log-trim-max-buckets 50 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 20 --debug-ms 0 2026-03-20T11:46:48.346 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:48.346 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:48.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.366+0000 7f25a9678900 20 rados->read ofs=0 len=0 2026-03-20T11:46:48.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.367+0000 7f25a9678900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:46:48.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.367+0000 7f25a9678900 20 rados->read ofs=0 len=0 2026-03-20T11:46:48.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.368+0000 7f25a9678900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:46:48.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.368+0000 7f25a9678900 20 realm test-realm 6a789b89-c2b8-45f5-82cc-fb80234d0c6c 2026-03-20T11:46:48.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.368+0000 7f25a9678900 20 rados->read ofs=0 len=0 2026-03-20T11:46:48.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.368+0000 7f25a9678900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:46:48.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.368+0000 7f25a9678900 20 rados->read ofs=0 len=0 2026-03-20T11:46:48.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.369+0000 7f25a9678900 20 rados_obj.operate() r=0 bl.length=10 2026-03-20T11:46:48.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.369+0000 7f25a9678900 20 rados->read ofs=0 len=0 2026-03-20T11:46:48.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.370+0000 7f25a9678900 20 rados_obj.operate() r=0 bl.length=2020 2026-03-20T11:46:48.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.370+0000 7f25a9678900 20 rados->read ofs=0 len=0 2026-03-20T11:46:48.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.370+0000 7f25a9678900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:46:48.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.370+0000 7f25a9678900 20 rados->read ofs=0 len=0 2026-03-20T11:46:48.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.371+0000 7f25a9678900 20 rados_obj.operate() r=0 bl.length=1030 2026-03-20T11:46:48.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.371+0000 7f25a9678900 20 zone a2 found 2026-03-20T11:46:48.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.371+0000 7f25a9678900 4 Realm: test-realm (6a789b89-c2b8-45f5-82cc-fb80234d0c6c) 2026-03-20T11:46:48.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.371+0000 7f25a9678900 4 ZoneGroup: a (a) 2026-03-20T11:46:48.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.371+0000 7f25a9678900 4 Zone: a2 (51055576-6662-409d-8abb-ecaeefd3dd80) 2026-03-20T11:46:48.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.371+0000 7f25a9678900 4 using period configuration: cb2fcdce-2c1b-429a-8601-a284ee3a2381:12 2026-03-20T11:46:48.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.371+0000 7f25a9678900 20 period zonegroup name a 2026-03-20T11:46:48.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.371+0000 7f25a9678900 20 using current period zonegroup a 2026-03-20T11:46:48.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.371+0000 7f25a9678900 20 rados->read ofs=0 len=0 2026-03-20T11:46:48.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.371+0000 7f25a9678900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.371+0000 7f25a9678900 20 rados->read ofs=0 len=0 2026-03-20T11:46:48.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.372+0000 7f25a9678900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.372+0000 7f25a9678900 20 init(): pipe_map (zone=4b634ebf-58f5-45bc-839a-cb4b425e773a bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:46:48.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.372+0000 7f25a9678900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.372+0000 7f25a9678900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.372+0000 7f25a9678900 20 rados->read ofs=0 len=0 2026-03-20T11:46:48.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.372+0000 7f25a9678900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.372+0000 7f25a9678900 20 rados->read ofs=0 len=0 2026-03-20T11:46:48.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.372+0000 7f25a9678900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.372+0000 7f25a9678900 20 init(): pipe_map (zone=51055576-6662-409d-8abb-ecaeefd3dd80 bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:46:48.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.372+0000 7f25a9678900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.372+0000 7f25a9678900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.372+0000 7f25a9678900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.372+0000 7f25a9678900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.372+0000 7f25a9678900 20 started sync module instance, tier type = 2026-03-20T11:46:48.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.372+0000 7f25a9678900 20 generating connection object for zone a1 id 4b634ebf-58f5-45bc-839a-cb4b425e773a 2026-03-20T11:46:48.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.372+0000 7f25a9678900 20 started zone id=51055576-6662-409d-8abb-ecaeefd3dd80 (name=a2) with tier type = 2026-03-20T11:46:48.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.376+0000 7f25a9678900 20 add_watcher() i=0 2026-03-20T11:46:48.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.376+0000 7f25a9678900 20 add_watcher() i=7 2026-03-20T11:46:48.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.376+0000 7f25a9678900 20 add_watcher() i=2 2026-03-20T11:46:48.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.377+0000 7f25a9678900 20 add_watcher() i=3 2026-03-20T11:46:48.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.377+0000 7f25a9678900 20 add_watcher() i=1 2026-03-20T11:46:48.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.378+0000 7f25a9678900 20 add_watcher() i=4 2026-03-20T11:46:48.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.378+0000 7f25a9678900 20 add_watcher() i=5 2026-03-20T11:46:48.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.378+0000 7f25a9678900 20 add_watcher() i=6 2026-03-20T11:46:48.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.378+0000 7f25a9678900 2 all 8 watchers are set, enabling cache 2026-03-20T11:46:48.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.380+0000 7f25a9678900 20 rgw_check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2026-03-20T11:46:48.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.380+0000 7f25a9678900 20 rgw_check_secure_mon_conn(): mode 1 is insecure 2026-03-20T11:46:48.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.380+0000 7f25a9678900 5 note: GC not initialized 2026-03-20T11:46:48.433 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.437+0000 7f25a9678900 20 init_complete bucket index max shards: 11 2026-03-20T11:46:48.433 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.437+0000 7f25a9678900 20 Filter name: none 2026-03-20T11:46:48.434 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.437+0000 7f2554fe1640 20 reqs_thread_entry: start 2026-03-20T11:46:48.434 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.437+0000 7f2506ffd640 20 reqs_thread_entry: start 2026-03-20T11:46:48.435 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.438+0000 7f25a9678900 10 rgw bucket trim manager: Watching bilog.trim 2026-03-20T11:46:48.435 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.438+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccb620:12BucketTrimCR: operate() 2026-03-20T11:46:48.435 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.438+0000 7f25a9678900 10 fetching active bucket counters 2026-03-20T11:46:48.435 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.438+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4cce0d0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:48.435 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.438+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4cce0d0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:48.435 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.438+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is io blocked 2026-03-20T11:46:48.435 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.438+0000 7f257effd640 20 trim: get_bucket_counters: [] 2026-03-20T11:46:48.435 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.439+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4cce0d0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:48.435 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.439+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4cce0d0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:48.436 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.439+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4cce0d0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:48.436 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.439+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4cce0d0:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:48.436 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.439+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccb620:12BucketTrimCR: operate() 2026-03-20T11:46:48.436 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.439+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:48.436 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.439+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:48.436 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.439+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is io blocked 2026-03-20T11:46:48.436 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.439+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:48.436 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.439+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:48.436 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.439+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:48.436 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.439+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:48.436 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.439+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccb620:12BucketTrimCR: operate() 2026-03-20T11:46:48.436 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.439+0000 7f25a9678900 10 listing cold buckets from marker=16:a04872b5:root::dipepu-33:head 2026-03-20T11:46:48.436 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.439+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:14MetadataListCR: operate() 2026-03-20T11:46:48.436 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.439+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:14MetadataListCR: operate() 2026-03-20T11:46:48.436 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.439+0000 7f25a9678900 20 enqueued request req=0x7f253c0014d0 2026-03-20T11:46:48.436 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.439+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.436 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.439+0000 7f25a9678900 20 req: 0x7f253c0014d0 2026-03-20T11:46:48.436 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.439+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is io blocked 2026-03-20T11:46:48.436 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.439+0000 7f256e7fc640 20 dequeued request req=0x7f253c0014d0 2026-03-20T11:46:48.436 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.439+0000 7f256e7fc640 20 RGWWQ: empty 2026-03-20T11:46:48.436 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.439+0000 7f256e7fc640 20 rgw async rados processor: starting metadata listing at 16:a04872b5:root::dipepu-33:head 2026-03-20T11:46:48.439 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.442+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-33 2026-03-20T11:46:48.439 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.442+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:48.441 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.445+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-4 2026-03-20T11:46:48.442 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.445+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:46:48.444 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.447+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:48.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.449+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:48.449 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.452+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-43 2026-03-20T11:46:48.449 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.452+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-30 2026-03-20T11:46:48.449 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.452+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-18 2026-03-20T11:46:48.449 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.452+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-2 2026-03-20T11:46:48.449 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.452+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:46:48.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.454+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-8 2026-03-20T11:46:48.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.454+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:46:48.453 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.456+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:48.454 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.458+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-58 2026-03-20T11:46:48.455 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.458+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:46:48.456 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.459+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:48.458 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.461+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-27 2026-03-20T11:46:48.458 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.461+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-28 2026-03-20T11:46:48.458 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.461+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-59 2026-03-20T11:46:48.458 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.461+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:46:48.459 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.462+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:48.461 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.464+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-40 2026-03-20T11:46:48.461 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.464+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-22 2026-03-20T11:46:48.461 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.464+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:46:48.462 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.465+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-56 2026-03-20T11:46:48.462 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.465+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:46:48.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.466+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:46:48.464 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.467+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:48.465 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.469+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-53 2026-03-20T11:46:48.465 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.469+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:46:48.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.470+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:46:48.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.471+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:46:48.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.471+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:46:48.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.472+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:46:48.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.473+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-34 2026-03-20T11:46:48.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.473+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-21 2026-03-20T11:46:48.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.473+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-14 2026-03-20T11:46:48.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.473+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-50 2026-03-20T11:46:48.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.473+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-57 2026-03-20T11:46:48.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.473+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:46:48.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.473+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-39 2026-03-20T11:46:48.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.473+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:48.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.473+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-37 2026-03-20T11:46:48.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.473+0000 7f256e7fc640 20 rgw async rados processor: restarting metadata listing 2026-03-20T11:46:48.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.480+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-10 2026-03-20T11:46:48.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.480+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:48.484 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.487+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:48.490 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.494+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:48.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.499+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:48.502 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.505+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:48.507 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.510+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:48.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.516+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-19 2026-03-20T11:46:48.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.516+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-3 2026-03-20T11:46:48.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.516+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:48.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.521+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:48.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.527+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-17 2026-03-20T11:46:48.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.527+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:48.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.533+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-38 2026-03-20T11:46:48.529 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.533+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:48.534 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.538+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-24 2026-03-20T11:46:48.535 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.538+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-36 2026-03-20T11:46:48.535 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.538+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:48.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.542+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:48.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.547+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:48.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.551+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-44 2026-03-20T11:46:48.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.551+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-35 2026-03-20T11:46:48.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.551+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:48.553 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.556+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-16 2026-03-20T11:46:48.553 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.556+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:48.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.560+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:48.562 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.565+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:48.567 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.570+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-55 2026-03-20T11:46:48.567 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.570+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-13 2026-03-20T11:46:48.567 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.570+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-48 2026-03-20T11:46:48.567 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.570+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-25 2026-03-20T11:46:48.567 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.570+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-51 2026-03-20T11:46:48.567 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.570+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:48.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.575+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:48.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.579+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-46 2026-03-20T11:46:48.576 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.579+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:46:48.581 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.584+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:46:48.588 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.592+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:46:48.593 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.596+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-26 2026-03-20T11:46:48.593 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.596+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-20 2026-03-20T11:46:48.593 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.596+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:46:48.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.601+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:46:48.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.606+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-1 2026-03-20T11:46:48.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.606+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:46:48.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.610+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:46:48.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f256e7fc640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:46:48.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:14MetadataListCR: operate() 2026-03-20T11:46:48.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:14MetadataListCR: operate() 2026-03-20T11:46:48.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:14MetadataListCR: operate() 2026-03-20T11:46:48.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:14MetadataListCR: operate() 2026-03-20T11:46:48.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccb620:12BucketTrimCR: operate() 2026-03-20T11:46:48.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 4 collected 50 buckets for trim 2026-03-20T11:46:48.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 cr:s=0x5636c4cd01a0:op=0x5636c4ccd0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 4 starting trim on bucket=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:46:48.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 cr:s=0x5636c4d0fd50:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 4 starting trim on bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:48.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 4 starting trim on bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:46:48.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 4 starting trim on bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:48.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 cr:s=0x5636c4cd01a0:op=0x5636c4cd45b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 cr:s=0x5636c4d0fd50:op=0x5636c4cd4d00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4cd5450:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd5ba0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 cr:s=0x5636c4cd01a0:op=0x5636c4cd45b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 enqueued request req=0x5636c4ccfca0 2026-03-20T11:46:48.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 req: 0x5636c4ccfca0 2026-03-20T11:46:48.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 run: stack=0x5636c4cd01a0 is io blocked 2026-03-20T11:46:48.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 cr:s=0x5636c4d0fd50:op=0x5636c4cd4d00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 enqueued request req=0x5636c4cd64a0 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 req: 0x5636c4ccfca0 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 req: 0x5636c4cd64a0 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 run: stack=0x5636c4d0fd50 is io blocked 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4cd5450:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 enqueued request req=0x5636c4cd6840 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 req: 0x5636c4ccfca0 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 req: 0x5636c4cd64a0 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 req: 0x5636c4cd6840 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 run: stack=0x5636c4c06270 is io blocked 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd5ba0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 enqueued request req=0x5636c4882640 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 req: 0x5636c4ccfca0 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 req: 0x5636c4cd64a0 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 req: 0x5636c4cd6840 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 req: 0x5636c4882640 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is io blocked 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f256cff9640 20 dequeued request req=0x5636c4ccfca0 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f256cff9640 20 RGWWQ: 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f256cff9640 20 req: 0x5636c4cd64a0 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f256cff9640 20 req: 0x5636c4cd6840 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f256cff9640 20 req: 0x5636c4882640 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f256cff9640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 : miss 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f256cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f2563fff640 20 dequeued request req=0x5636c4cd64a0 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f2563fff640 20 RGWWQ: 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f2563fff640 20 req: 0x5636c4cd6840 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f2563fff640 20 req: 0x5636c4882640 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f2563fff640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 : miss 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f2563fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f256dffb640 20 dequeued request req=0x5636c4cd6840 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f256dffb640 20 RGWWQ: 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f256dffb640 20 req: 0x5636c4882640 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f256dffb640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 : miss 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f256d7fa640 20 dequeued request req=0x5636c4882640 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f256d7fa640 20 RGWWQ: empty 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f256dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f256d7fa640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 : miss 2026-03-20T11:46:48.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.614+0000 7f256d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.613 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256d7fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.613 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256d7fa640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 info.flags=0x17 2026-03-20T11:46:48.613 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256d7fa640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 to cache LRU end 2026-03-20T11:46:48.613 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256d7fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.613 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256d7fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.613 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256d7fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:48.613 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256d7fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-20 : miss 2026-03-20T11:46:48.613 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.613 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256dffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.613 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256dffb640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 info.flags=0x17 2026-03-20T11:46:48.613 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256dffb640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 to cache LRU end 2026-03-20T11:46:48.613 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256dffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.613 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256dffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256dffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256dffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-30 : miss 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f2563fff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f2563fff640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 info.flags=0x17 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f2563fff640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 to cache LRU end 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f2563fff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f2563fff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f2563fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f2563fff640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-58 : miss 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f2563fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256cff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256cff9640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 info.flags=0x17 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256cff9640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 to cache LRU end 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256cff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256cff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256cff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256cff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-61 : miss 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256dffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256dffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-30 info.flags=0x0 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256dffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-30 to cache LRU end 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256dffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-30 : miss 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f2563fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f2563fff640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-58 info.flags=0x0 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f2563fff640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-58 to cache LRU end 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f2563fff640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-58 : miss 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f2563fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256cff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256cff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-61 info.flags=0x0 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256cff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-61 to cache LRU end 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256cff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-61 : miss 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256d7fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256d7fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-20 info.flags=0x0 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256d7fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-20 to cache LRU end 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256d7fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-20 : miss 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.615+0000 7f256d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f256dffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f256dffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-30 info.flags=0x0 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f256dffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-30 to cache LRU end 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f256dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35): adding source pipe: {s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f256dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35): adding dest pipe: {s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f256dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f256dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f256dffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f256d7fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f256d7fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-20 info.flags=0x0 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f256d7fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-20 to cache LRU end 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f256d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25): adding source pipe: {s={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f256d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25): adding dest pipe: {s={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4cd5450:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4cd5450:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4cd5450:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4cd5450:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f256d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f256d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4882a60:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f256d7fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 run: stack=0x5636c4c06270 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4883080:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd5ba0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4883080:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f2563fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f2563fff640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-58 info.flags=0x0 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f2563fff640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-58 to cache LRU end 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f2563fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18): adding source pipe: {s={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f2563fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18): adding dest pipe: {s={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f2563fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f2563fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 9fa138bd0c595efad755895ca4b8001f8ccd7bc3df9593afd68941c4ea7d84cd 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f2563fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:9fa138bd0c595efad755895ca4b8001f8ccd7bc3df9593afd68941c4ea7d84cd 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = c4e07a6cc1cb09233e993328338039317bf97a6291ff8b3d8b26c3eabacaacdd 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c4e07a6cc1cb09233e993328338039317bf97a6291ff8b3d8b26c3eabacaacdd 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=0, curl_handle=0x5636c4884b90 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is io blocked 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd5ba0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4d0fd50:op=0x5636c4cd4d00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd5ba0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4d0fd50:op=0x5636c4cd4d00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd5ba0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4d0fd50:op=0x5636c4cd4d00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.615 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4d0fd50:op=0x5636c4cd4d00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4cd6160:op=0x5636c4885b00:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4d0fd50:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4cd6160:op=0x5636c4cd4d00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c4cd8120:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4d0fd50:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 run: stack=0x5636c4d0fd50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4cd6160:op=0x5636c4cd4d00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f2506ffd640 20 link_request req_data=0x5636c4885650 req_data->id=0, curl_handle=0x5636c4884b90 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 977883416f908f3c591dceafc109164875e0c6b797af18dc66a12e923d9d2a3c 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:977883416f908f3c591dceafc109164875e0c6b797af18dc66a12e923d9d2a3c 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 850517d18d58594f8946a675995be491b854b9b4f4425d94348cba0fa64cb761 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=850517d18d58594f8946a675995be491b854b9b4f4425d94348cba0fa64cb761 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=1, curl_handle=0x5636c4cd9c00 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 run: stack=0x5636c4cd6160 is io blocked 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c4cd8740:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c4cd8740:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 3bd4e6e90c09f2c97a6040d827abc718fc91347fcf512846cb50f39a182acb73 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f2506ffd640 20 link_request req_data=0x5636c4cda310 req_data->id=1, curl_handle=0x5636c4cd9c00 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:3bd4e6e90c09f2c97a6040d827abc718fc91347fcf512846cb50f39a182acb73 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 8297f78aadb219081bc5b60095f97d5349d3265c223d4a54ad8995c3a0cf69e1 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=8297f78aadb219081bc5b60095f97d5349d3265c223d4a54ad8995c3a0cf69e1 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=2, curl_handle=0x5636c4cdd120 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 run: stack=0x5636c48865d0 is io blocked 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f2506ffd640 20 link_request req_data=0x5636c4cdd870 req_data->id=2, curl_handle=0x5636c4cdd120 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f256cff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f256cff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-61 info.flags=0x0 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f256cff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-61 to cache LRU end 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f256cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21): adding source pipe: {s={b=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f256cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21): adding dest pipe: {s={b=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f256cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f256cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f256cff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4cd01a0:op=0x5636c4cd45b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4cd01a0:op=0x5636c4cd45b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4cd01a0:op=0x5636c4cd45b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4cd01a0:op=0x5636c4cd45b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4cd01a0:op=0x5636c4ccd0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.617 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4cdf260:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4cd01a0:op=0x5636c4ccd0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 run: stack=0x5636c4cd01a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4cdfa10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4cdfa10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 8643506f065398dde7d5f7f25310f389a9f398c8face1810d865725a3bfb6f1e 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:8643506f065398dde7d5f7f25310f389a9f398c8face1810d865725a3bfb6f1e 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = b007c322fc70aedc79f6f661c67f62b01ae5243b92be1e095461847c8bbf56d8 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b007c322fc70aedc79f6f661c67f62b01ae5243b92be1e095461847c8bbf56d8 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=3, curl_handle=0x5636c4ce15f0 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is io blocked 2026-03-20T11:46:48.618 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.616+0000 7f2506ffd640 20 link_request req_data=0x5636c4ce1d00 req_data->id=3, curl_handle=0x5636c4ce15f0 2026-03-20T11:46:48.618 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:48.617+0000 7f12d7a9f640 -1 req 6760126013744735873 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4883080:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4883080:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4882a60:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 15 stack 0x5636c4cd5900 end 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is done 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c4cd8740:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4cdfa10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c4cd6160:op=0x5636c4cd4d00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 collect(): s=0x5636c4c06270 stack=0x5636c4cd5900 is complete 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 15 stack 0x5636c4c06270 end 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 stack->operate() returned ret=-22 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 run: stack=0x5636c4c06270 is done 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c4cd8740:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4cdfa10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c4cd6160:op=0x5636c4cd4d00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c4cd8740:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4cdfa10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c4cd6160:op=0x5636c4cd4d00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 4 starting trim on bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c4cd8740:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4cdfa10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c4cd6160:op=0x5636c4cd4d00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4ce77b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.620 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c4cd8120:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 15 stack 0x5636c48865d0 end 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 run: stack=0x5636c48865d0 is done 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4cdf260:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 15 stack 0x5636c4cdf880 end 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is done 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c4cd6160:op=0x5636c4885b00:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 15 stack 0x5636c4cd6160 end 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 run: stack=0x5636c4cd6160 is done 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4ce77b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 enqueued request req=0x5636c4cce570 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 req: 0x5636c4cce570 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 run: stack=0x5636c4c06270 is io blocked 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c4d0fd50:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 collect(): s=0x5636c4d0fd50 stack=0x5636c48865d0 is complete 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) markers=[,00000000001.98.6,,,,00000000001.102.6,,,00000000001.40.6,,00000000001.48.6], shards=11 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c4cd01a0:op=0x5636c4ccd0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 collect(): s=0x5636c4cd01a0 stack=0x5636c4cdf880 is complete 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-61[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21]) markers=[,00000000001.75.6,,,,00000000001.74.6,,,00000000001.106.6,,00000000001.81.6], shards=11 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 collect(): s=0x5636c4cce390 stack=0x5636c4cd6160 is complete 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-20[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25]) markers=[,,,,,,,,00000000001.41.14,,], shards=11 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c4d0fd50:op=0x5636c4d3a2d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 10 trimming bilog shard 1 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.98.6 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 10 trimming bilog shard 5 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.102.6 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 10 trimming bilog shard 8 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.40.6 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 10 trimming bilog shard 10 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.48.6 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 run: stack=0x5636c4d0fd50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 20 cr:s=0x5636c4cd01a0:op=0x5636c4cdf240:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 10 trimming bilog shard 1 of :dipepu-61[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21]) at marker 00000000001.75.6 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.619+0000 7f25a9678900 10 trimming bilog shard 5 of :dipepu-61[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21]) at marker 00000000001.74.6 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 10 trimming bilog shard 8 of :dipepu-61[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21]) at marker 00000000001.106.6 2026-03-20T11:46:48.621 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 10 trimming bilog shard 10 of :dipepu-61[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21]) at marker 00000000001.81.6 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 run: stack=0x5636c4cd01a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4885ae0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 10 trimming bilog shard 8 of :dipepu-20[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25]) at marker 00000000001.41.14 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 cr:s=0x5636c4cd6160:op=0x5636c4cef030:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d397f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c48829c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d3ad30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 cr:s=0x5636c4ce8680:op=0x5636c4ce3820:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4cdf9e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 cr:s=0x5636c4cd9990:op=0x5636c4ced670:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 cr:s=0x5636c4d40270:op=0x5636c4d40440:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d40c40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 cr:s=0x5636c4cd6160:op=0x5636c4cef030:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.1 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 run: stack=0x5636c4cd6160 is io blocked 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d397f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.5 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is io blocked 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c48829c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.8 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 run: stack=0x5636c48865d0 is io blocked 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d3ad30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.10 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is io blocked 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 cr:s=0x5636c4ce8680:op=0x5636c4ce3820:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.1 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 run: stack=0x5636c4ce8680 is io blocked 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4cdf9e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.5 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 run: stack=0x5636c4d41490 is io blocked 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 cr:s=0x5636c4cd9990:op=0x5636c4ced670:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.8 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f2556fe5640 20 dequeued request req=0x5636c4cce570 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f2556fe5640 20 RGWWQ: empty 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f2556fe5640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 : miss 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 run: stack=0x5636c4cd9990 is io blocked 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 cr:s=0x5636c4d40270:op=0x5636c4d40440:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.622 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f2556fe5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.10 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 run: stack=0x5636c4d40270 is io blocked 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d40c40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25.8 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.620+0000 7f25a9678900 20 run: stack=0x5636c4884340 is io blocked 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c48829c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c48829c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 15 stack 0x5636c48865d0 end 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 run: stack=0x5636c48865d0 is done 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4d0fd50:op=0x5636c4d3a2d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 run: stack=0x5636c4d0fd50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4cd6160:op=0x5636c4cef030:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4cd6160:op=0x5636c4cef030:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 15 stack 0x5636c4cd6160 end 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 run: stack=0x5636c4cd6160 is done 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4d0fd50:op=0x5636c4d3a2d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 run: stack=0x5636c4d0fd50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4d40270:op=0x5636c4d40440:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4d40270:op=0x5636c4d40440:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 15 stack 0x5636c4d40270 end 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 run: stack=0x5636c4d40270 is done 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4cd01a0:op=0x5636c4cdf240:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 run: stack=0x5636c4cd01a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f2556fe5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f2556fe5640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 info.flags=0x17 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f2556fe5640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 to cache LRU end 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f2556fe5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f2556fe5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f2556fe5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f2556fe5640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-52 : miss 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f2556fe5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d40c40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d40c40:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 15 stack 0x5636c4884340 end 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 run: stack=0x5636c4884340 is done 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4885ae0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 15 stack 0x5636c4cce390 end 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is done 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 4 starting trim on bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd4d00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4ce8680:op=0x5636c4ce3820:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4ce8680:op=0x5636c4ce3820:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 15 stack 0x5636c4ce8680 end 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 run: stack=0x5636c4ce8680 is done 2026-03-20T11:46:48.623 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4cdf9e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4cdf9e0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 15 stack 0x5636c4d41490 end 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 run: stack=0x5636c4d41490 is done 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd4d00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 enqueued request req=0x5636c4cd45f0 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 req: 0x5636c4cd45f0 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is io blocked 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4cd01a0:op=0x5636c4cdf240:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4cd01a0:op=0x5636c4cdf240:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 run: stack=0x5636c4cd01a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25637fe640 20 dequeued request req=0x5636c4cd45f0 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25637fe640 20 RGWWQ: empty 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25637fe640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 : miss 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25637fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4cd9990:op=0x5636c4ced670:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4cd9990:op=0x5636c4ced670:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 15 stack 0x5636c4cd9990 end 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 run: stack=0x5636c4cd9990 is done 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4cd01a0:op=0x5636c4cdf240:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d397f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d397f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 15 stack 0x5636c4cdf880 end 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f2556fe5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f2556fe5640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-52 info.flags=0x0 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f2556fe5640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-52 to cache LRU end 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is done 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4cd01a0:op=0x5636c4ccd0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 15 stack 0x5636c4cd01a0 end 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 run: stack=0x5636c4cd01a0 is done 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4d0fd50:op=0x5636c4d3a2d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 run: stack=0x5636c4d0fd50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4ccd0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 4 starting trim on bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c48829c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c48829c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 enqueued request req=0x5636c4883ca0 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 req: 0x5636c4883ca0 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is io blocked 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f2556fe5640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-52 : miss 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f2556fe5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f2562ffd640 20 dequeued request req=0x5636c4883ca0 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f2562ffd640 20 RGWWQ: empty 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f2562ffd640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 : miss 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f2562ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d3ad30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d3ad30:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 15 stack 0x5636c4cd5900 end 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is done 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4d0fd50:op=0x5636c4d3a2d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4d0fd50:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.624 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:48.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 15 stack 0x5636c4d0fd50 end 2026-03-20T11:46:48.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 run: stack=0x5636c4d0fd50 is done 2026-03-20T11:46:48.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 4 starting trim on bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:48.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cef030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cef030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 enqueued request req=0x5636c4ccfa40 2026-03-20T11:46:48.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 req: 0x5636c4ccfa40 2026-03-20T11:46:48.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is io blocked 2026-03-20T11:46:48.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25627fc640 20 dequeued request req=0x5636c4ccfa40 2026-03-20T11:46:48.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25627fc640 20 RGWWQ: empty 2026-03-20T11:46:48.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25627fc640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 : miss 2026-03-20T11:46:48.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.622+0000 7f25627fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25637fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:48.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25637fe640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 info.flags=0x17 2026-03-20T11:46:48.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25637fe640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 to cache LRU end 2026-03-20T11:46:48.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25637fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25637fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25637fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:46:48.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25637fe640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-42 : miss 2026-03-20T11:46:48.625 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25637fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f2562ffd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f2562ffd640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 info.flags=0x17 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f2562ffd640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 to cache LRU end 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f2562ffd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f2562ffd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f2562ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f2562ffd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-2 : miss 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f2562ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25627fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25627fc640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 info.flags=0x17 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25627fc640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 to cache LRU end 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25627fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25627fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25627fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25627fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-57 : miss 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25627fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f2556fe5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f2556fe5640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-52 info.flags=0x0 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f2556fe5640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-52 to cache LRU end 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f2556fe5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12): adding source pipe: {s={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f2556fe5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12): adding dest pipe: {s={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f2556fe5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f2556fe5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f2556fe5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4ce77b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4ce77b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4ce77b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4ce77b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25a9678900 20 cr:s=0x5636c4cd9990:op=0x5636c4cdd1b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25a9678900 20 run: stack=0x5636c4c06270 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25a9678900 20 cr:s=0x5636c4cd9990:op=0x5636c4ce47c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25a9678900 20 cr:s=0x5636c4cd9990:op=0x5636c4ce47c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25627fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25627fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-57 info.flags=0x0 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25627fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-57 to cache LRU end 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25627fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-57 : miss 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25627fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25637fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25637fe640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-42 info.flags=0x0 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25637fe640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-42 to cache LRU end 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25637fe640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-42 : miss 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25637fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.626 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f2562ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f2562ffd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-2 info.flags=0x0 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f2562ffd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-2 to cache LRU end 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f2562ffd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-2 : miss 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f2562ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = d5614d737b78cd25b2af444aa6d3f6a906f590c578f56927a56a6834d3a36be7 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:d5614d737b78cd25b2af444aa6d3f6a906f590c578f56927a56a6834d3a36be7 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 031b30ce4cde8117a50c08efef513e56369829373c4544de53416958ed8017df 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=031b30ce4cde8117a50c08efef513e56369829373c4544de53416958ed8017df 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25637fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25637fe640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-42 info.flags=0x0 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25627fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25637fe640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-42 to cache LRU end 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25627fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-57 info.flags=0x0 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25627fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-57 to cache LRU end 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25637fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2): adding source pipe: {s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25627fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17): adding source pipe: {s={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25627fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17): adding dest pipe: {s={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25637fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2): adding dest pipe: {s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25627fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25627fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25637fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25637fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25627fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.623+0000 7f25637fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=4, curl_handle=0x5636c4cd9c00 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 run: stack=0x5636c4cd9990 is io blocked 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cef030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd4d00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.627 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cef030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd4d00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cef030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd4d00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cef030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd4d00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f2562ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f2562ffd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-2 info.flags=0x0 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f2562ffd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-2 to cache LRU end 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4885ae0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f2562ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2): adding source pipe: {s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cec970:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f2562ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2): adding dest pipe: {s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4cd4d00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4ce77b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4cd4d00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f2562ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f2562ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f2562ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f2506ffd640 20 link_request req_data=0x5636c4cda310 req_data->id=4, curl_handle=0x5636c4cd9c00 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 38d01cbf3d355f3bbdc939454e0678b8ed0a570d7282dba582f2be9f56fbd89e 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:38d01cbf3d355f3bbdc939454e0678b8ed0a570d7282dba582f2be9f56fbd89e 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = fe96a74690ee83d992400c18edfc68da24b2450f40aac2bd0c49e74b875fa74e 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=fe96a74690ee83d992400c18edfc68da24b2450f40aac2bd0c49e74b875fa74e 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.628 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=5, curl_handle=0x5636c4ce15f0 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 run: stack=0x5636c4d41490 is io blocked 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4ce77b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f2506ffd640 20 link_request req_data=0x5636c4886970 req_data->id=5, curl_handle=0x5636c4ce15f0 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 4aed754db3e49e4263b15e04fb70b370715e0441ceef6295fdd6a55008e9ef27 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:4aed754db3e49e4263b15e04fb70b370715e0441ceef6295fdd6a55008e9ef27 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 210afacc9d43349fd6012fa749cc03b7e822f54bbb0c7784f968523db1c401f9 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=210afacc9d43349fd6012fa749cc03b7e822f54bbb0c7784f968523db1c401f9 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=6, curl_handle=0x5636c4cdd120 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 run: stack=0x5636c4884340 is io blocked 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c48829c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c48829c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c48829c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c48829c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f2506ffd640 20 link_request req_data=0x5636c4ce1d00 req_data->id=6, curl_handle=0x5636c4cdd120 2026-03-20T11:46:48.629 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.624+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4ccd0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.629 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:48.625+0000 7f1212114640 -1 req 11787068841075615956 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.625+0000 7f25a9678900 20 cr:s=0x5636c4d40270:op=0x5636c4ce01f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.625+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4ccd0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.625+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.625+0000 7f25a9678900 20 cr:s=0x5636c4d40270:op=0x5636c48829c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.625+0000 7f25a9678900 20 cr:s=0x5636c4d40270:op=0x5636c48829c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.625+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.625+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.625+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.625+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.625+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.625+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.625+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 2903dd94e4a1a6e451ac93cf44551c17bd12d59618612fc905de61634ce495ae 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.626+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:2903dd94e4a1a6e451ac93cf44551c17bd12d59618612fc905de61634ce495ae 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.626+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.626+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.626+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.630 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.626+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.626+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = eff9407977e18402b821d09d6d8d18a0d1d92736a4fc50ec11d1c29685dd6acf 2026-03-20T11:46:48.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.626+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=eff9407977e18402b821d09d6d8d18a0d1d92736a4fc50ec11d1c29685dd6acf 2026-03-20T11:46:48.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.626+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.626+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.626+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.626+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=7, curl_handle=0x5636c4884b90 2026-03-20T11:46:48.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.626+0000 7f25a9678900 20 run: stack=0x5636c4d40270 is io blocked 2026-03-20T11:46:48.631 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:48.626+0000 7f12cba87640 -1 req 15015204596205696411 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:48.631 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.626+0000 7f2506ffd640 20 link_request req_data=0x5636c4cdd870 req_data->id=7, curl_handle=0x5636c4884b90 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4cd4d00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4cd4d00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4ce77b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4ce77b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4cd4d00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cec970:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 15 stack 0x5636c4884340 end 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 run: stack=0x5636c4884340 is done 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4cd4d00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 collect(): s=0x5636c4cce390 stack=0x5636c4884340 is complete 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 15 stack 0x5636c4cce390 end 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 stack->operate() returned ret=-22 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is done 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4885ae0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 15 stack 0x5636c4d41490 end 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 run: stack=0x5636c4d41490 is done 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 collect(): s=0x5636c4cd5900 stack=0x5636c4d41490 is complete 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-57[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17]) markers=[,00000000001.63.6,,,,00000000001.43.6,,,,,], shards=11 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 4 starting trim on bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4885ae0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.663 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 10 trimming bilog shard 1 of :dipepu-57[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17]) at marker 00000000001.63.6 2026-03-20T11:46:48.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 10 trimming bilog shard 5 of :dipepu-57[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17]) at marker 00000000001.43.6 2026-03-20T11:46:48.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4870cc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4cdf9c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4ce3820:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4870cc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 enqueued request req=0x5636c4882640 2026-03-20T11:46:48.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 req: 0x5636c4882640 2026-03-20T11:46:48.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is io blocked 2026-03-20T11:46:48.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4cdf9c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f2561ffb640 20 dequeued request req=0x5636c4882640 2026-03-20T11:46:48.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f2561ffb640 20 RGWWQ: empty 2026-03-20T11:46:48.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f2561ffb640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 : miss 2026-03-20T11:46:48.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17.1 2026-03-20T11:46:48.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f2561ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 run: stack=0x5636c4d41490 is io blocked 2026-03-20T11:46:48.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4ce3820:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17.5 2026-03-20T11:46:48.664 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f25a9678900 20 run: stack=0x5636c4884340 is io blocked 2026-03-20T11:46:48.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f2561ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f2561ffb640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 info.flags=0x17 2026-03-20T11:46:48.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f2561ffb640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 to cache LRU end 2026-03-20T11:46:48.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f2561ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f2561ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:48.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f2561ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:46:48.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f2561ffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-29 : miss 2026-03-20T11:46:48.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.666+0000 7f2561ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f2561ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f2561ffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-29 info.flags=0x0 2026-03-20T11:46:48.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f2561ffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-29 to cache LRU end 2026-03-20T11:46:48.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f2561ffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-29 : miss 2026-03-20T11:46:48.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f2561ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4cd9990:op=0x5636c4ce47c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f2561ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f2561ffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-29 info.flags=0x0 2026-03-20T11:46:48.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f2561ffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-29 to cache LRU end 2026-03-20T11:46:48.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f2561ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34): adding source pipe: {s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f2561ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34): adding dest pipe: {s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f2561ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f2561ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4cd9990:op=0x5636c4ce47c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4d40270:op=0x5636c48829c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4d40270:op=0x5636c48829c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:48.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4cd9990:op=0x5636c4ce47c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4d40270:op=0x5636c4ce01f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 15 stack 0x5636c4d40270 end 2026-03-20T11:46:48.665 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 run: stack=0x5636c4d40270 is done 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4cd9990:op=0x5636c4ce47c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4ccd0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 collect(): s=0x5636c4cdf880 stack=0x5636c4d40270 is complete 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4ccd0c0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 15 stack 0x5636c4cdf880 end 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f2561ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 stack->operate() returned ret=-22 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is done 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4cd9990:op=0x5636c4cdd1b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 15 stack 0x5636c4cd9990 end 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 run: stack=0x5636c4cd9990 is done 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 collect(): s=0x5636c4c06270 stack=0x5636c4cd9990 is complete 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) markers=[00000000002.30.6,,00000000002.60.6,,00000000002.47.6,,,00000000002.30.6,,,], shards=11 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4ccd0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 4 starting trim on bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4870cc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4d3e9d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 10 trimming bilog shard 0 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.30.6 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 10 trimming bilog shard 2 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.60.6 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 10 trimming bilog shard 4 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.47.6 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 10 trimming bilog shard 7 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.30.6 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 run: stack=0x5636c4c06270 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4873690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4870cc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4cd9990:op=0x5636c48829c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4d40270:op=0x5636c4d3ad30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4cd6160:op=0x5636c4cec7a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c4d40440:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4873690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 enqueued request req=0x5636c4d439f0 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 req: 0x5636c4d439f0 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is io blocked 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4870cc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25617fa640 20 dequeued request req=0x5636c4d439f0 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25617fa640 20 RGWWQ: empty 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25617fa640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 : miss 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25617fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4cd9990:op=0x5636c48829c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.0 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 run: stack=0x5636c4cd9990 is io blocked 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4ce3820:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4ce3820:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 15 stack 0x5636c4884340 end 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 run: stack=0x5636c4884340 is done 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4d40270:op=0x5636c4d3ad30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.2 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 run: stack=0x5636c4d40270 is io blocked 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4cd6160:op=0x5636c4cec7a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.4 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 run: stack=0x5636c4cd6160 is io blocked 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c4d40440:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.7 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 run: stack=0x5636c48865d0 is io blocked 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4870cc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4885ae0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdf240:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = e352c8beaccaf7a37acf1486d2212786f8d4429a0a373005df68db38e2f2acb0 2026-03-20T11:46:48.667 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:e352c8beaccaf7a37acf1486d2212786f8d4429a0a373005df68db38e2f2acb0 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = e3a23a089ac3618449a390876cc46a0a8f67c5fa3442dee03994e666f1c8b14f 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=e3a23a089ac3618449a390876cc46a0a8f67c5fa3442dee03994e666f1c8b14f 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=8, curl_handle=0x5636c4884b90 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 run: stack=0x5636c4884340 is io blocked 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4cdf9c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4cdf9c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 15 stack 0x5636c4d41490 end 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f2506ffd640 20 link_request req_data=0x5636c4cdd870 req_data->id=8, curl_handle=0x5636c4884b90 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 run: stack=0x5636c4d41490 is done 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4885ae0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 15 stack 0x5636c4cd5900 end 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is done 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 4 starting trim on bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4870ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4870ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 enqueued request req=0x5636c4cdcee0 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 req: 0x5636c4cdcee0 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is io blocked 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f2560ff9640 20 dequeued request req=0x5636c4cdcee0 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.667+0000 7f2560ff9640 20 RGWWQ: empty 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.668+0000 7f2560ff9640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 : miss 2026-03-20T11:46:48.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.668+0000 7f2560ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.669 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:48.668+0000 7f11fa0e4640 -1 req 4612954963560214140 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.668+0000 7f25617fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.668+0000 7f25617fa640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 info.flags=0x17 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.668+0000 7f25617fa640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 to cache LRU end 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.668+0000 7f25617fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.668+0000 7f25617fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.668+0000 7f25617fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.668+0000 7f25617fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-23 : miss 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.668+0000 7f25617fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.668+0000 7f2560ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.668+0000 7f2560ff9640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 info.flags=0x17 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.668+0000 7f2560ff9640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 to cache LRU end 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.668+0000 7f2560ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.668+0000 7f2560ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.668+0000 7f2560ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.668+0000 7f2560ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.668+0000 7f2560ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-34 : miss 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.668+0000 7f2560ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.668+0000 7f25a9678900 20 cr:s=0x5636c4cd9990:op=0x5636c48829c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.668+0000 7f25a9678900 20 cr:s=0x5636c4cd9990:op=0x5636c48829c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.668+0000 7f25a9678900 15 stack 0x5636c4cd9990 end 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.668+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.668+0000 7f25a9678900 20 run: stack=0x5636c4cd9990 is done 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.668+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4d3e9d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.668+0000 7f25a9678900 20 run: stack=0x5636c4c06270 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.668+0000 7f25617fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.668+0000 7f25617fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-23 info.flags=0x0 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25617fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-23 to cache LRU end 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25617fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-23 : miss 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25617fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c4d40270:op=0x5636c4d3ad30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c4d40270:op=0x5636c4d3ad30:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 15 stack 0x5636c4d40270 end 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 run: stack=0x5636c4d40270 is done 2026-03-20T11:46:48.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4d3e9d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 run: stack=0x5636c4c06270 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c4cd6160:op=0x5636c4cec7a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c4cd6160:op=0x5636c4cec7a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 15 stack 0x5636c4cd6160 end 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 run: stack=0x5636c4cd6160 is done 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c4d40440:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c4d40440:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 15 stack 0x5636c48865d0 end 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 run: stack=0x5636c48865d0 is done 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4d3e9d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4d3e9d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 15 stack 0x5636c4c06270 end 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 run: stack=0x5636c4c06270 is done 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 4 starting trim on bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4cef760:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4cef760:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 enqueued request req=0x5636c4ccffb0 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 req: 0x5636c4ccffb0 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25607f8640 20 dequeued request req=0x5636c4ccffb0 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25607f8640 20 RGWWQ: empty 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 run: stack=0x5636c4c06270 is io blocked 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25607f8640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 : miss 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25607f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f2560ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f2560ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-34 info.flags=0x0 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f2560ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-34 to cache LRU end 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f2560ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-34 : miss 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f2560ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25617fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25617fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-23 info.flags=0x0 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25617fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-23 to cache LRU end 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25617fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28): adding source pipe: {s={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25617fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28): adding dest pipe: {s={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25617fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25617fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.671 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25617fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4873690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4873690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4873690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4873690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4ccd0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25607f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25607f8640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 info.flags=0x17 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c4d39990:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25607f8640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 to cache LRU end 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25607f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25607f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4ccd0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c4ce47c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c4ce47c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25607f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25607f8640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-54 : miss 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25607f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = d074a0fa8de6b803c242c036812f73226d375edfb2cedececaf99581250a152f 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:d074a0fa8de6b803c242c036812f73226d375edfb2cedececaf99581250a152f 2026-03-20T11:46:48.672 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 5c3401fd6cf8f3a2abaf024aa2355e88231acc55c56b3067a37ccd9e66c5efcf 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5c3401fd6cf8f3a2abaf024aa2355e88231acc55c56b3067a37ccd9e66c5efcf 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f2560ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f2560ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-34 info.flags=0x0 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f2560ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-34 to cache LRU end 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f2560ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39): adding source pipe: {s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f2560ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39): adding dest pipe: {s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f2560ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f2560ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f2560ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=9, curl_handle=0x5636c4cd9c00 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 run: stack=0x5636c48865d0 is io blocked 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4870ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4870ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4870ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4870ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c4cd6160:op=0x5636c4d3a2a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c4cd6160:op=0x5636c4870ca0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 cr:s=0x5636c4cd6160:op=0x5636c4870ca0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25607f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25607f8640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-54 info.flags=0x0 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25607f8640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-54 to cache LRU end 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25607f8640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-54 : miss 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25607f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f2506ffd640 20 link_request req_data=0x5636c4cda310 req_data->id=9, curl_handle=0x5636c4cd9c00 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.673 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 97f1ffd4a44107908f62f40ad7000185157d93a07139d585338277fec4c47a0f 2026-03-20T11:46:48.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.674 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.674 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.674 INFO:teuthology.orchestra.run.vm02.stderr:97f1ffd4a44107908f62f40ad7000185157d93a07139d585338277fec4c47a0f 2026-03-20T11:46:48.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.669+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.674 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:48.671+0000 7f12cd28a640 -1 req 7660086489055143275 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:48.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.670+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 42c41aa68bde65eb9f05077278edfa1f5ef6d4c1a42ce040531050cdd96770c6 2026-03-20T11:46:48.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.670+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=42c41aa68bde65eb9f05077278edfa1f5ef6d4c1a42ce040531050cdd96770c6 2026-03-20T11:46:48.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.670+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.670+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.670+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.670+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=10, curl_handle=0x5636c4cdd120 2026-03-20T11:46:48.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.670+0000 7f25a9678900 20 run: stack=0x5636c4cd6160 is io blocked 2026-03-20T11:46:48.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.670+0000 7f2506ffd640 20 link_request req_data=0x5636c4ce1d00 req_data->id=10, curl_handle=0x5636c4cdd120 2026-03-20T11:46:48.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.670+0000 7f25607f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.670+0000 7f25607f8640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-54 info.flags=0x0 2026-03-20T11:46:48.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.670+0000 7f25607f8640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-54 to cache LRU end 2026-03-20T11:46:48.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.670+0000 7f25607f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14): adding source pipe: {s={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.670+0000 7f25607f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14): adding dest pipe: {s={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.670+0000 7f25607f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.670+0000 7f25607f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25607f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:48.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4cef760:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4cef760:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.674 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4cef760:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4cef760:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 20 cr:s=0x5636c4d40270:op=0x5636c4cef760:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 20 run: stack=0x5636c4c06270 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 20 cr:s=0x5636c4d40270:op=0x5636c48762d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 20 cr:s=0x5636c4d40270:op=0x5636c48762d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 5a8eeec325fa682475801b4bc74f6ac27f5b426e32688e7c46f472a34f113c72 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:5a8eeec325fa682475801b4bc74f6ac27f5b426e32688e7c46f472a34f113c72 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = b0d99530396b1cd30ca6786995c356ca8f658d8004e2a9032b02b0599a825e1a 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b0d99530396b1cd30ca6786995c356ca8f658d8004e2a9032b02b0599a825e1a 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=11, curl_handle=0x5636c4ce15f0 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f25a9678900 20 run: stack=0x5636c4d40270 is io blocked 2026-03-20T11:46:48.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.671+0000 7f2506ffd640 20 link_request req_data=0x5636c4886970 req_data->id=11, curl_handle=0x5636c4ce15f0 2026-03-20T11:46:48.706 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdf240:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f25a9678900 15 stack 0x5636c4884340 end 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f25a9678900 20 run: stack=0x5636c4884340 is done 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f25a9678900 20 collect(): s=0x5636c4cce390 stack=0x5636c4884340 is complete 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f25a9678900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f25a9678900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f25a9678900 15 stack 0x5636c4cce390 end 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f25a9678900 20 stack->operate() returned ret=-22 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is done 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f25a9678900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f25a9678900 4 starting trim on bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4873630:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4873630:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f25a9678900 20 enqueued request req=0x5636c4882640 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f25a9678900 20 req: 0x5636c4882640 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is io blocked 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f255fff7640 20 dequeued request req=0x5636c4882640 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f255fff7640 20 RGWWQ: empty 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f255fff7640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 : miss 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f255fff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f255fff7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f255fff7640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 info.flags=0x17 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f255fff7640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 to cache LRU end 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f255fff7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f255fff7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f255fff7640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f255fff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f255fff7640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-32 : miss 2026-03-20T11:46:48.707 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.709+0000 7f255fff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.710+0000 7f255fff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.710+0000 7f255fff7640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-32 info.flags=0x0 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.710+0000 7f255fff7640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-32 to cache LRU end 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.710+0000 7f255fff7640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-32 : miss 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.710+0000 7f255fff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.710+0000 7f255fff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.710+0000 7f255fff7640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-32 info.flags=0x0 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.710+0000 7f255fff7640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-32 to cache LRU end 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.710+0000 7f255fff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37): adding source pipe: {s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.710+0000 7f255fff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37): adding dest pipe: {s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.710+0000 7f255fff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.710+0000 7f255fff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.710+0000 7f255fff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.710+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4873630:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.710+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4873630:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.710+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4873630:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.710+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4873630:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.710+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.710+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdf240:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.710+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.710+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.710+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.710+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.710+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.710+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.710+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.710+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.711+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.711+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.711+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = ec3d3f38388f00cea353aa80c55b4072fe642fac365e010e49ebead8e3b0e2ef 2026-03-20T11:46:48.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.711+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.709 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.709 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.709 INFO:teuthology.orchestra.run.vm02.stderr:ec3d3f38388f00cea353aa80c55b4072fe642fac365e010e49ebead8e3b0e2ef 2026-03-20T11:46:48.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.711+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.711+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.711+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.711+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.711+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 5bee449b325c5f7a3198738b1d39b3da2817839698437e0a8509ec9fd8b8eceb 2026-03-20T11:46:48.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.711+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5bee449b325c5f7a3198738b1d39b3da2817839698437e0a8509ec9fd8b8eceb 2026-03-20T11:46:48.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.711+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.711+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.711+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.711+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=12, curl_handle=0x5636c4884b90 2026-03-20T11:46:48.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.711+0000 7f25a9678900 20 run: stack=0x5636c4884340 is io blocked 2026-03-20T11:46:48.709 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.711+0000 7f2506ffd640 20 link_request req_data=0x5636c4cdd870 req_data->id=12, curl_handle=0x5636c4884b90 2026-03-20T11:46:48.709 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:48.711+0000 7f12f52da640 -1 req 7775767504441825704 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c4ce47c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c4ce47c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 cr:s=0x5636c4cd6160:op=0x5636c4870ca0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 cr:s=0x5636c4cd6160:op=0x5636c4870ca0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c4ce47c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 cr:s=0x5636c4cd6160:op=0x5636c4d3a2a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 15 stack 0x5636c4cd6160 end 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 run: stack=0x5636c4cd6160 is done 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c4ce47c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 collect(): s=0x5636c4cd5900 stack=0x5636c4cd6160 is complete 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 15 stack 0x5636c4cd5900 end 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 stack->operate() returned ret=-22 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is done 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c4d39990:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 15 stack 0x5636c48865d0 end 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 run: stack=0x5636c48865d0 is done 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4ccd0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 collect(): s=0x5636c4cdf880 stack=0x5636c48865d0 is complete 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-23[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28]) markers=[,,,,,,,,00000000002.27.13,,], shards=11 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 4 starting trim on bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d39990:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 10 trimming bilog shard 8 of :dipepu-23[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28]) at marker 00000000002.27.13 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.710 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4ceb070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c4878780:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4ceb070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 enqueued request req=0x5636c4883ca0 2026-03-20T11:46:48.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 req: 0x5636c4883ca0 2026-03-20T11:46:48.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is io blocked 2026-03-20T11:46:48.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c4878780:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28.8 2026-03-20T11:46:48.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f25a9678900 20 run: stack=0x5636c48865d0 is io blocked 2026-03-20T11:46:48.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f255f7f6640 20 dequeued request req=0x5636c4883ca0 2026-03-20T11:46:48.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f255f7f6640 20 RGWWQ: empty 2026-03-20T11:46:48.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f255f7f6640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 : miss 2026-03-20T11:46:48.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f255f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f255f7f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f255f7f6640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 info.flags=0x17 2026-03-20T11:46:48.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f255f7f6640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 to cache LRU end 2026-03-20T11:46:48.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f255f7f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f255f7f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f255f7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:46:48.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f255f7f6640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-19 : miss 2026-03-20T11:46:48.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f255f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f255f7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f255f7f6640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-19 info.flags=0x0 2026-03-20T11:46:48.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f255f7f6640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-19 to cache LRU end 2026-03-20T11:46:48.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f255f7f6640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-19 : miss 2026-03-20T11:46:48.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.712+0000 7f255f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4d40270:op=0x5636c48762d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f255f7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f255f7f6640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-19 info.flags=0x0 2026-03-20T11:46:48.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f255f7f6640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-19 to cache LRU end 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4d40270:op=0x5636c48762d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f255f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24): adding source pipe: {s={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4d40270:op=0x5636c48762d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4d40270:op=0x5636c48762d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4d40270:op=0x5636c4cef760:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 15 stack 0x5636c4d40270 end 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 run: stack=0x5636c4d40270 is done 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 collect(): s=0x5636c4c06270 stack=0x5636c4d40270 is complete 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f255f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24): adding dest pipe: {s={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) markers=[00000000004.65.1,00000000002.57.1,00000000004.95.1,00000000002.36.1,00000000004.41.1,00000000002.67.1,00000000002.53.1,00000000004.82.1,00000000002.67.1,00000000002.123.1,00000000002.39.1], shards=11 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f255f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4885ae0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 10 trimming bilog shard 0 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.65.1 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 10 trimming bilog shard 1 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.57.1 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 10 trimming bilog shard 2 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.95.1 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 10 trimming bilog shard 3 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.36.1 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f255f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 10 trimming bilog shard 4 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.41.1 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 10 trimming bilog shard 5 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.67.1 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 10 trimming bilog shard 6 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.53.1 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 10 trimming bilog shard 7 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.82.1 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f255f7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 10 trimming bilog shard 8 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.67.1 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 10 trimming bilog shard 9 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.123.1 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 10 trimming bilog shard 10 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.39.1 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 run: stack=0x5636c4c06270 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4d40270:op=0x5636c4ce37b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4cd6160:op=0x5636c4d403f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4cd9990:op=0x5636c4ce6d30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4d37e00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4873db0:op=0x5636c4872720:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d3c600:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4d43d20:op=0x5636c4d41c70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4874190:op=0x5636c4ce5600:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4cd8560:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4875da0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4873f10:op=0x5636c4874c00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4ceb070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4d40270:op=0x5636c4ce37b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.0 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 run: stack=0x5636c4d40270 is io blocked 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4cd6160:op=0x5636c4d403f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.1 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 run: stack=0x5636c4cd6160 is io blocked 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4cd9990:op=0x5636c4ce6d30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.2 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 run: stack=0x5636c4cd9990 is io blocked 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4d37e00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.3 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 run: stack=0x5636c4d41490 is io blocked 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4873db0:op=0x5636c4872720:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.4 2026-03-20T11:46:48.715 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 run: stack=0x5636c4873db0 is io blocked 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d3c600:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.5 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is io blocked 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4d43d20:op=0x5636c4d41c70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.6 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 run: stack=0x5636c4d43d20 is io blocked 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4874190:op=0x5636c4ce5600:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.7 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 run: stack=0x5636c4874190 is io blocked 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4cd8560:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.8 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 run: stack=0x5636c4cee790 is io blocked 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4875da0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.9 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 run: stack=0x5636c48737c0 is io blocked 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4873f10:op=0x5636c4874c00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.10 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 run: stack=0x5636c4873f10 is io blocked 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4ceb070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4ceb070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4ceb070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4ce9bc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d3a260:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d3a260:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 2d2c18b85c522136d60a1e968ff90ded7f36e2c07192967c57525305de1f374e 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2d2c18b85c522136d60a1e968ff90ded7f36e2c07192967c57525305de1f374e 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 24e911b373d5dd6fd4235be61de330b6d23b98846ef01e346a9bae00cbb4c610 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=24e911b373d5dd6fd4235be61de330b6d23b98846ef01e346a9bae00cbb4c610 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=13, curl_handle=0x5636c4ce15f0 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 run: stack=0x5636c4873b00 is io blocked 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c4878780:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c4878780:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 15 stack 0x5636c48865d0 end 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 run: stack=0x5636c48865d0 is done 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d39990:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4ccd0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 15 stack 0x5636c4cdf880 end 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is done 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f2506ffd640 20 link_request req_data=0x5636c4886970 req_data->id=13, curl_handle=0x5636c4ce15f0 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4ccd0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 4 starting trim on bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4ceb070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4ceb070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 enqueued request req=0x5636c48850c0 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 req: 0x5636c48850c0 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is io blocked 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f255eff5640 20 dequeued request req=0x5636c48850c0 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f255eff5640 20 RGWWQ: empty 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f255eff5640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 : miss 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.713+0000 7f255eff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4875da0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4875da0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 15 stack 0x5636c48737c0 end 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 run: stack=0x5636c48737c0 is done 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4885ae0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 run: stack=0x5636c4c06270 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f255eff5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f255eff5640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 info.flags=0x17 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f255eff5640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 to cache LRU end 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f255eff5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f255eff5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c4873f10:op=0x5636c4874c00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c4873f10:op=0x5636c4874c00:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 15 stack 0x5636c4873f10 end 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 run: stack=0x5636c4873f10 is done 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4885ae0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 run: stack=0x5636c4c06270 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c4874190:op=0x5636c4ce5600:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c4874190:op=0x5636c4ce5600:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 15 stack 0x5636c4874190 end 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 run: stack=0x5636c4874190 is done 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d3c600:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d3c600:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 15 stack 0x5636c4d3ab70 end 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is done 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4cd8560:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4cd8560:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 15 stack 0x5636c4cee790 end 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 run: stack=0x5636c4cee790 is done 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4885ae0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4885ae0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.717 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4885ae0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 run: stack=0x5636c4c06270 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4d37e00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4d37e00:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 15 stack 0x5636c4d41490 end 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 run: stack=0x5636c4d41490 is done 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4885ae0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 run: stack=0x5636c4c06270 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c4cd9990:op=0x5636c4ce6d30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c4cd9990:op=0x5636c4ce6d30:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 15 stack 0x5636c4cd9990 end 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 run: stack=0x5636c4cd9990 is done 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4885ae0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 run: stack=0x5636c4c06270 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f255eff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f255eff5640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-4 : miss 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f255eff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c4d40270:op=0x5636c4ce37b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c4d40270:op=0x5636c4ce37b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 15 stack 0x5636c4d40270 end 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 run: stack=0x5636c4d40270 is done 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4885ae0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 run: stack=0x5636c4c06270 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c4d43d20:op=0x5636c4d41c70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c4d43d20:op=0x5636c4d41c70:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 15 stack 0x5636c4d43d20 end 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 run: stack=0x5636c4d43d20 is done 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c4cd6160:op=0x5636c4d403f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c4cd6160:op=0x5636c4d403f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 15 stack 0x5636c4cd6160 end 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 run: stack=0x5636c4cd6160 is done 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4885ae0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4885ae0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.716+0000 7f25a9678900 20 run: stack=0x5636c4c06270 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255eff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255eff5640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-4 info.flags=0x0 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255eff5640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-4 to cache LRU end 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255eff5640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-4 : miss 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255eff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 cr:s=0x5636c4873db0:op=0x5636c4872720:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 cr:s=0x5636c4873db0:op=0x5636c4872720:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 15 stack 0x5636c4873db0 end 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 run: stack=0x5636c4873db0 is done 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4885ae0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 cr:s=0x5636c4c06270:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:48.718 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 15 stack 0x5636c4c06270 end 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 run: stack=0x5636c4c06270 is done 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 4 starting trim on bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4ce5600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4ce5600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 enqueued request req=0x5636c4cea950 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 req: 0x5636c4cea950 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255e7f4640 20 dequeued request req=0x5636c4cea950 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255e7f4640 20 RGWWQ: empty 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255e7f4640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 : miss 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255e7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 run: stack=0x5636c4d41490 is io blocked 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255e7f4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255e7f4640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 info.flags=0x17 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255e7f4640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 to cache LRU end 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255e7f4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255e7f4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255e7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255e7f4640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-53 : miss 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255e7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255eff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255eff5640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-4 info.flags=0x0 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255eff5640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-4 to cache LRU end 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255eff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5): adding source pipe: {s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255eff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5): adding dest pipe: {s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255e7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255e7f4640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-53 info.flags=0x0 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255e7f4640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-53 to cache LRU end 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255e7f4640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-53 : miss 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255e7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255eff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255eff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f255eff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4ceb070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4ceb070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4ceb070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4ceb070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4ccd0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d39990:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4ccd0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4885ae0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4885ae0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.719 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.720 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.720 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.720 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.720 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.720 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.720 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.720 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = ff89bda3995938e02163fcd720bfa1904781fcf91d902758d751371ae7b14f6e 2026-03-20T11:46:48.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.720 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.720 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.720 INFO:teuthology.orchestra.run.vm02.stderr:ff89bda3995938e02163fcd720bfa1904781fcf91d902758d751371ae7b14f6e 2026-03-20T11:46:48.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = f1817105f00e7ef3b4e9cf190ad747e92826dad911ce1eb9a0b82c532c438ca8 2026-03-20T11:46:48.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=f1817105f00e7ef3b4e9cf190ad747e92826dad911ce1eb9a0b82c532c438ca8 2026-03-20T11:46:48.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=14, curl_handle=0x5636c4cdd120 2026-03-20T11:46:48.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f25a9678900 20 run: stack=0x5636c4cee790 is io blocked 2026-03-20T11:46:48.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.717+0000 7f2506ffd640 20 link_request req_data=0x5636c4ce1d00 req_data->id=14, curl_handle=0x5636c4cdd120 2026-03-20T11:46:48.720 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:48.718+0000 7f1230150640 -1 req 904460173812372611 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:48.720 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f255e7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f255e7f4640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-53 info.flags=0x0 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f255e7f4640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-53 to cache LRU end 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f255e7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13): adding source pipe: {s={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f255e7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13): adding dest pipe: {s={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f255e7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f255e7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f255e7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4ce5600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4ce5600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4ce5600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4ce5600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 20 cr:s=0x5636c4874190:op=0x5636c4d42580:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 20 run: stack=0x5636c4d41490 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 20 cr:s=0x5636c4874190:op=0x5636c4ceb000:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 20 cr:s=0x5636c4874190:op=0x5636c4ceb000:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 2fb5d0f86b9cf52c4ebd3682a821772bdfddcf614868d9a7306b66a7e2a1f283 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2fb5d0f86b9cf52c4ebd3682a821772bdfddcf614868d9a7306b66a7e2a1f283 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = b8749a198b47c1132d4f933153f3f8ea590637a3b26d2bff5cd62cd6b5db2d50 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b8749a198b47c1132d4f933153f3f8ea590637a3b26d2bff5cd62cd6b5db2d50 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=15, curl_handle=0x5636c4cd9c00 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f25a9678900 20 run: stack=0x5636c4874190 is io blocked 2026-03-20T11:46:48.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.718+0000 7f2506ffd640 20 link_request req_data=0x5636c4cda310 req_data->id=15, curl_handle=0x5636c4cd9c00 2026-03-20T11:46:48.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:48.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdf240:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f25a9678900 15 stack 0x5636c4884340 end 2026-03-20T11:46:48.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f25a9678900 20 run: stack=0x5636c4884340 is done 2026-03-20T11:46:48.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f25a9678900 20 collect(): s=0x5636c4cce390 stack=0x5636c4884340 is complete 2026-03-20T11:46:48.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f25a9678900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:48.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f25a9678900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:48.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:48.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f25a9678900 15 stack 0x5636c4cce390 end 2026-03-20T11:46:48.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f25a9678900 20 stack->operate() returned ret=-22 2026-03-20T11:46:48.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is done 2026-03-20T11:46:48.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f25a9678900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:48.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f25a9678900 4 starting trim on bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:46:48.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f25a9678900 20 enqueued request req=0x5636c4882640 2026-03-20T11:46:48.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f25a9678900 20 req: 0x5636c4882640 2026-03-20T11:46:48.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f255dff3640 20 dequeued request req=0x5636c4882640 2026-03-20T11:46:48.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f255dff3640 20 RGWWQ: empty 2026-03-20T11:46:48.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f255dff3640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 : miss 2026-03-20T11:46:48.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f255dff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is io blocked 2026-03-20T11:46:48.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f255dff3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f255dff3640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 info.flags=0x17 2026-03-20T11:46:48.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f255dff3640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 to cache LRU end 2026-03-20T11:46:48.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f255dff3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f255dff3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f255dff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:46:48.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f255dff3640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-14 : miss 2026-03-20T11:46:48.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f255dff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f255dff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f255dff3640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-14 info.flags=0x0 2026-03-20T11:46:48.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f255dff3640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-14 to cache LRU end 2026-03-20T11:46:48.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f255dff3640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-14 : miss 2026-03-20T11:46:48.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.752+0000 7f255dff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f255dff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f255dff3640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-14 info.flags=0x0 2026-03-20T11:46:48.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f255dff3640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-14 to cache LRU end 2026-03-20T11:46:48.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f255dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19): adding source pipe: {s={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f255dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19): adding dest pipe: {s={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.750 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f255dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f255dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f255dff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cebca0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = f72820697a61e7a6989c0100c11e5f04ef86db2b9bb4b6079ca0ab08479904bf 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:f72820697a61e7a6989c0100c11e5f04ef86db2b9bb4b6079ca0ab08479904bf 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = daf7ba59afb749584e510ca29fcca0df56ab81e51abfa0f07a0232895368438e 2026-03-20T11:46:48.751 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=daf7ba59afb749584e510ca29fcca0df56ab81e51abfa0f07a0232895368438e 2026-03-20T11:46:48.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=16, curl_handle=0x5636c4884b90 2026-03-20T11:46:48.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f25a9678900 20 run: stack=0x5636c4884340 is io blocked 2026-03-20T11:46:48.752 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.753+0000 7f2506ffd640 20 link_request req_data=0x5636c4cdd870 req_data->id=16, curl_handle=0x5636c4884b90 2026-03-20T11:46:48.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.757+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d3a260:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.757+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d3a260:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.757+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d3a260:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.757+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d3a260:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.757+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4ce9bc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.757+0000 7f25a9678900 15 stack 0x5636c4873b00 end 2026-03-20T11:46:48.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.757+0000 7f25a9678900 20 run: stack=0x5636c4873b00 is done 2026-03-20T11:46:48.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.757+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.757+0000 7f25a9678900 20 collect(): s=0x5636c4cd5900 stack=0x5636c4873b00 is complete 2026-03-20T11:46:48.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.757+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.757+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-19[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24]) markers=[,,,,,,,,00000000005.35.12,,], shards=11 2026-03-20T11:46:48.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.757+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d44500:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.757+0000 7f25a9678900 10 trimming bilog shard 8 of :dipepu-19[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24]) at marker 00000000005.35.12 2026-03-20T11:46:48.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.757+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.757+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4ce9ad0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.757+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4ce9ad0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.757+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24.8 2026-03-20T11:46:48.754 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.757+0000 7f25a9678900 20 run: stack=0x5636c4873b00 is io blocked 2026-03-20T11:46:48.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4ce9ad0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4ce9ad0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f25a9678900 15 stack 0x5636c4873b00 end 2026-03-20T11:46:48.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f25a9678900 20 run: stack=0x5636c4873b00 is done 2026-03-20T11:46:48.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d44500:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:46:48.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f25a9678900 15 stack 0x5636c4cd5900 end 2026-03-20T11:46:48.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is done 2026-03-20T11:46:48.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f25a9678900 4 starting trim on bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:46:48.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d42fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d42fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f25a9678900 20 enqueued request req=0x5636c4883ca0 2026-03-20T11:46:48.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f25a9678900 20 req: 0x5636c4883ca0 2026-03-20T11:46:48.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f255d7f2640 20 dequeued request req=0x5636c4883ca0 2026-03-20T11:46:48.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f255d7f2640 20 RGWWQ: empty 2026-03-20T11:46:48.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f255d7f2640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 : miss 2026-03-20T11:46:48.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f255d7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is io blocked 2026-03-20T11:46:48.755 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f255d7f2640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f255d7f2640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 info.flags=0x17 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f255d7f2640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 to cache LRU end 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f255d7f2640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f255d7f2640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f255d7f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f255d7f2640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-36 : miss 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f255d7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f255d7f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f255d7f2640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-36 info.flags=0x0 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f255d7f2640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-36 to cache LRU end 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f255d7f2640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-36 : miss 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.758+0000 7f255d7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4885ae0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4885ae0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d39990:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f25a9678900 15 stack 0x5636c4cee790 end 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f25a9678900 20 run: stack=0x5636c4cee790 is done 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4ccd0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f25a9678900 20 collect(): s=0x5636c4cdf880 stack=0x5636c4cee790 is complete 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f25a9678900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f25a9678900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4ccd0c0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f25a9678900 15 stack 0x5636c4cdf880 end 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f25a9678900 20 stack->operate() returned ret=-22 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is done 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f25a9678900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4ccd0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f25a9678900 4 starting trim on bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f25a9678900 20 enqueued request req=0x5636c48850c0 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f25a9678900 20 req: 0x5636c48850c0 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is io blocked 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f255cff1640 20 dequeued request req=0x5636c48850c0 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f255cff1640 20 RGWWQ: empty 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f255cff1640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 : miss 2026-03-20T11:46:48.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f255cff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f255cff1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f255cff1640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 info.flags=0x17 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f255cff1640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 to cache LRU end 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f255cff1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f255cff1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f255cff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f255cff1640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-22 : miss 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.759+0000 7f255cff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4874190:op=0x5636c4ceb000:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4874190:op=0x5636c4ceb000:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4874190:op=0x5636c4ceb000:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4874190:op=0x5636c4ceb000:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4874190:op=0x5636c4d42580:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 15 stack 0x5636c4874190 end 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 run: stack=0x5636c4874190 is done 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 collect(): s=0x5636c4d41490 stack=0x5636c4874190 is complete 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) markers=[00000000004.87.6,00000000002.126.1,00000000004.75.6,00000000002.34.1,00000000004.89.6,00000000002.87.1,00000000002.128.1,00000000004.94.6,00000000002.37.1,00000000002.73.1,00000000002.61.1], shards=11 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4d38710:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 10 trimming bilog shard 0 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.87.6 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 10 trimming bilog shard 1 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.126.1 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 10 trimming bilog shard 2 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.75.6 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 10 trimming bilog shard 3 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.34.1 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 10 trimming bilog shard 4 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.89.6 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 10 trimming bilog shard 5 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.87.1 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 10 trimming bilog shard 6 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.128.1 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 10 trimming bilog shard 7 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.94.6 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 10 trimming bilog shard 8 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.37.1 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 10 trimming bilog shard 9 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.73.1 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 10 trimming bilog shard 10 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.61.1 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 run: stack=0x5636c4d41490 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4874190:op=0x5636c4d41c70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4876160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4ce37b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d40180:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4873f10:op=0x5636c4874cd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4cd8120:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c4d42580:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4d3c5f0:op=0x5636c4cdfb90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4cee900:op=0x5636c4ceb000:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d48520:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.757 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d450c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4874190:op=0x5636c4d41c70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.0 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 run: stack=0x5636c4874190 is io blocked 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4876160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.1 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 run: stack=0x5636c4cee790 is io blocked 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4ce37b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.2 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 run: stack=0x5636c4873b00 is io blocked 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d40180:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.3 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is io blocked 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4873f10:op=0x5636c4874cd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.4 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 run: stack=0x5636c4873f10 is io blocked 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4cd8120:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.5 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 run: stack=0x5636c48737c0 is io blocked 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c4d42580:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.6 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 run: stack=0x5636c48865d0 is io blocked 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4d3c5f0:op=0x5636c4cdfb90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.7 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 run: stack=0x5636c4d3c5f0 is io blocked 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4cee900:op=0x5636c4ceb000:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.8 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 run: stack=0x5636c4cee900 is io blocked 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d48520:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.9 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 run: stack=0x5636c4ce56a0 is io blocked 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d450c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.10 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 run: stack=0x5636c4d3c450 is io blocked 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f255d7f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f255d7f2640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-36 info.flags=0x0 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f255d7f2640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-36 to cache LRU end 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f255d7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41): adding source pipe: {s={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f255d7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41): adding dest pipe: {s={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f255d7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f255d7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f255d7f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d42fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.758 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d42fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d42fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d42fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d48d20:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4cef030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4cef030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.760+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.761+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.761+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.761+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.761+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.761+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.761+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 5df541af4b48ac5a8df07fa05c0c593f94d3cb97b46867573eb77203c014a73c 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.761+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:5df541af4b48ac5a8df07fa05c0c593f94d3cb97b46867573eb77203c014a73c 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.761+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.761+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.761+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.761+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.761+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 84227c8de4dea216cab8d0d93f9cae0e70892d4541d399fc1922a8ab22e837c5 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.761+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=84227c8de4dea216cab8d0d93f9cae0e70892d4541d399fc1922a8ab22e837c5 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.761+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.761+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.761+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.761+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=17, curl_handle=0x5636c4cd9c00 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.761+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is io blocked 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.761+0000 7f2506ffd640 20 link_request req_data=0x5636c4cda310 req_data->id=17, curl_handle=0x5636c4cd9c00 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f255cff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f255cff1640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-22 info.flags=0x0 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f255cff1640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-22 to cache LRU end 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f25a9678900 20 cr:s=0x5636c4873f10:op=0x5636c4874cd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f25a9678900 20 cr:s=0x5636c4873f10:op=0x5636c4874cd0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f25a9678900 15 stack 0x5636c4873f10 end 2026-03-20T11:46:48.759 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f25a9678900 20 run: stack=0x5636c4873f10 is done 2026-03-20T11:46:48.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4d38710:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f25a9678900 20 run: stack=0x5636c4d41490 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d40180:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d40180:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f25a9678900 15 stack 0x5636c4d3ab70 end 2026-03-20T11:46:48.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is done 2026-03-20T11:46:48.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4876160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4876160:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f25a9678900 15 stack 0x5636c4cee790 end 2026-03-20T11:46:48.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f25a9678900 20 run: stack=0x5636c4cee790 is done 2026-03-20T11:46:48.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4cd8120:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4cd8120:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f25a9678900 15 stack 0x5636c48737c0 end 2026-03-20T11:46:48.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f25a9678900 20 run: stack=0x5636c48737c0 is done 2026-03-20T11:46:48.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4d38710:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4d38710:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4d38710:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.762+0000 7f25a9678900 20 run: stack=0x5636c4d41490 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f255cff1640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-22 : miss 2026-03-20T11:46:48.760 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f255cff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d48520:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d48520:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 15 stack 0x5636c4ce56a0 end 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 run: stack=0x5636c4ce56a0 is done 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4d38710:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 run: stack=0x5636c4d41490 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d450c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d450c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 15 stack 0x5636c4d3c450 end 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 run: stack=0x5636c4d3c450 is done 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4ce37b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4ce37b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 15 stack 0x5636c4873b00 end 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 run: stack=0x5636c4873b00 is done 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4d38710:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4d38710:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 run: stack=0x5636c4d41490 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 cr:s=0x5636c4874190:op=0x5636c4d41c70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 cr:s=0x5636c4874190:op=0x5636c4d41c70:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 15 stack 0x5636c4874190 end 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 run: stack=0x5636c4874190 is done 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4d38710:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 run: stack=0x5636c4d41490 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 cr:s=0x5636c4d3c5f0:op=0x5636c4cdfb90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 cr:s=0x5636c4d3c5f0:op=0x5636c4cdfb90:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 15 stack 0x5636c4d3c5f0 end 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 run: stack=0x5636c4d3c5f0 is done 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4d38710:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 run: stack=0x5636c4d41490 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c4d42580:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 cr:s=0x5636c48865d0:op=0x5636c4d42580:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 15 stack 0x5636c48865d0 end 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 run: stack=0x5636c48865d0 is done 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4d38710:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 run: stack=0x5636c4d41490 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 cr:s=0x5636c4cee900:op=0x5636c4ceb000:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 cr:s=0x5636c4cee900:op=0x5636c4ceb000:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 15 stack 0x5636c4cee900 end 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 run: stack=0x5636c4cee900 is done 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4d38710:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 cr:s=0x5636c4d41490:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 15 stack 0x5636c4d41490 end 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 run: stack=0x5636c4d41490 is done 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 4 starting trim on bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4882a00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4882a00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 enqueued request req=0x5636c4ccffb0 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 req: 0x5636c4ccffb0 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f25a9678900 20 run: stack=0x5636c4d3c450 is io blocked 2026-03-20T11:46:48.761 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f255c7f0640 20 dequeued request req=0x5636c4ccffb0 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f255c7f0640 20 RGWWQ: empty 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f255c7f0640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 : miss 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f255c7f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f255cff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f255cff1640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-22 info.flags=0x0 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f255cff1640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-22 to cache LRU end 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f255cff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27): adding source pipe: {s={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f255cff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27): adding dest pipe: {s={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f255cff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f255cff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f255c7f0640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f255c7f0640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 info.flags=0x17 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f255c7f0640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 to cache LRU end 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f255c7f0640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f255c7f0640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f255c7f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f255c7f0640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-25 : miss 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f255c7f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.763+0000 7f255cff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4ccd0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d39990:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4ccd0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 92a08f8e3aacfee5f153afe297f4290fbd35573730ec66eaee340b71a798f9f1 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:92a08f8e3aacfee5f153afe297f4290fbd35573730ec66eaee340b71a798f9f1 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = c1e76366501c7b7d0d07669e391978019349f36af66e1ef78290cdfe8ed757e7 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c1e76366501c7b7d0d07669e391978019349f36af66e1ef78290cdfe8ed757e7 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.762 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=18, curl_handle=0x5636c4cdd120 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f25a9678900 20 run: stack=0x5636c4873b00 is io blocked 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f2506ffd640 20 link_request req_data=0x5636c4ce1d00 req_data->id=18, curl_handle=0x5636c4cdd120 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f255c7f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f255c7f0640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-25 info.flags=0x0 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f255c7f0640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-25 to cache LRU end 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f255c7f0640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-25 : miss 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.764+0000 7f255c7f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f255c7f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f255c7f0640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-25 info.flags=0x0 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f255c7f0640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-25 to cache LRU end 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f255c7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30): adding source pipe: {s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f255c7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30): adding dest pipe: {s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f255c7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f255c7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f255c7f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4882a00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4882a00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4882a00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4882a00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d44500:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 20 run: stack=0x5636c4d3c450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d3e7a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d3e7a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.763 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 31045614d5108c4dfa1f99743488b6a622d483523383fc0fed4f9fc6109e64c2 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:31045614d5108c4dfa1f99743488b6a622d483523383fc0fed4f9fc6109e64c2 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 896e857145fdaf2c1936afdd03c74b1d1948c367cc568679f2278f7444067cc3 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=896e857145fdaf2c1936afdd03c74b1d1948c367cc568679f2278f7444067cc3 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=19, curl_handle=0x5636c4ce15f0 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f2506ffd640 20 link_request req_data=0x5636c4886970 req_data->id=19, curl_handle=0x5636c4ce15f0 2026-03-20T11:46:48.764 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.765+0000 7f25a9678900 20 run: stack=0x5636c4ce56a0 is io blocked 2026-03-20T11:46:48.764 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:48.765+0000 7f12e6abd640 -1 req 7122883293538030505 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:48.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.795+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.795+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.795+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.795+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.795+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cebca0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.795+0000 7f25a9678900 15 stack 0x5636c4884340 end 2026-03-20T11:46:48.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.795+0000 7f25a9678900 20 run: stack=0x5636c4884340 is done 2026-03-20T11:46:48.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.795+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.795+0000 7f25a9678900 20 collect(): s=0x5636c4cce390 stack=0x5636c4884340 is complete 2026-03-20T11:46:48.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.795+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.795+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) markers=[,,,,,00000000003.36.10,,,00000000011.47.6,00000000003.43.6,], shards=11 2026-03-20T11:46:48.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.795+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd8310:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.795+0000 7f25a9678900 10 trimming bilog shard 5 of :dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) at marker 00000000003.36.10 2026-03-20T11:46:48.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.795+0000 7f25a9678900 10 trimming bilog shard 8 of :dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) at marker 00000000011.47.6 2026-03-20T11:46:48.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.795+0000 7f25a9678900 10 trimming bilog shard 9 of :dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) at marker 00000000003.43.6 2026-03-20T11:46:48.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.795+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.795+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d41c70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.795+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4ce37b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.795+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d48520:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.795+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d41c70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.795+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19.5 2026-03-20T11:46:48.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.795+0000 7f25a9678900 20 run: stack=0x5636c4884340 is io blocked 2026-03-20T11:46:48.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.795+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4ce37b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.795+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19.8 2026-03-20T11:46:48.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.795+0000 7f25a9678900 20 run: stack=0x5636c48737c0 is io blocked 2026-03-20T11:46:48.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.795+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d48520:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.795+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19.9 2026-03-20T11:46:48.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.795+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is io blocked 2026-03-20T11:46:48.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4ce37b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4ce37b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 15 stack 0x5636c48737c0 end 2026-03-20T11:46:48.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 run: stack=0x5636c48737c0 is done 2026-03-20T11:46:48.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd8310:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d48520:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d48520:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 15 stack 0x5636c4d3ab70 end 2026-03-20T11:46:48.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is done 2026-03-20T11:46:48.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd8310:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d41c70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.793 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d41c70:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 15 stack 0x5636c4884340 end 2026-03-20T11:46:48.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 run: stack=0x5636c4884340 is done 2026-03-20T11:46:48.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd8310:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:46:48.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 15 stack 0x5636c4cce390 end 2026-03-20T11:46:48.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is done 2026-03-20T11:46:48.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 4 starting trim on bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:46:48.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 enqueued request req=0x5636c4882640 2026-03-20T11:46:48.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 req: 0x5636c4882640 2026-03-20T11:46:48.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is io blocked 2026-03-20T11:46:48.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f255bfef640 20 dequeued request req=0x5636c4882640 2026-03-20T11:46:48.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f255bfef640 20 RGWWQ: empty 2026-03-20T11:46:48.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f255bfef640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 : miss 2026-03-20T11:46:48.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.796+0000 7f255bfef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f255bfef640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f255bfef640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 info.flags=0x17 2026-03-20T11:46:48.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f255bfef640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 to cache LRU end 2026-03-20T11:46:48.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f255bfef640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.794 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f255bfef640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f255bfef640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f255bfef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f255bfef640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-35 : miss 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f255bfef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f255bfef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f255bfef640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-35 info.flags=0x0 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f255bfef640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-35 to cache LRU end 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f255bfef640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-35 : miss 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f255bfef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f255bfef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f255bfef640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-35 info.flags=0x0 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f255bfef640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-35 to cache LRU end 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f255bfef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40): adding source pipe: {s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f255bfef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40): adding dest pipe: {s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f255bfef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f255bfef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f255bfef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cebca0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.797+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.798+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.798+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.798+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.798+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.798+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.798+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 5c245694d5c45db7deba3368309769a792ee8a46754a365a95c2a961585cc234 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.798+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:5c245694d5c45db7deba3368309769a792ee8a46754a365a95c2a961585cc234 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.798+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.798+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.798+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.798+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.798+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 7c90aecfe140cf58878c23d633cb33f9626915f4eda32a8f52552bf1aeecc893 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.798+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=7c90aecfe140cf58878c23d633cb33f9626915f4eda32a8f52552bf1aeecc893 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.798+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.798+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.795 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.798+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.798+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=20, curl_handle=0x5636c4884b90 2026-03-20T11:46:48.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.798+0000 7f2506ffd640 20 link_request req_data=0x5636c4cdd870 req_data->id=20, curl_handle=0x5636c4884b90 2026-03-20T11:46:48.796 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:48.798+0000 7f123595b640 -1 req 11981234675782904463 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:48.796 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.798+0000 7f25a9678900 20 run: stack=0x5636c4884340 is io blocked 2026-03-20T11:46:48.800 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.804+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4cef030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.804+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4cef030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.804+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4cef030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.804+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4cef030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.804+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d48d20:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.804+0000 7f25a9678900 15 stack 0x5636c4d32ac0 end 2026-03-20T11:46:48.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.804+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is done 2026-03-20T11:46:48.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.804+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.804+0000 7f25a9678900 20 collect(): s=0x5636c4cd5900 stack=0x5636c4d32ac0 is complete 2026-03-20T11:46:48.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.804+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.804+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-36[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41]) markers=[,,,,,,,,,,00000000001.53.6], shards=11 2026-03-20T11:46:48.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.804+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cef030:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.804+0000 7f25a9678900 10 trimming bilog shard 10 of :dipepu-36[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41]) at marker 00000000001.53.6 2026-03-20T11:46:48.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.804+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.804+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4ce5ff0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.804+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4ce5ff0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.804+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41.10 2026-03-20T11:46:48.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.804+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is io blocked 2026-03-20T11:46:48.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4ce5ff0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4ce5ff0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.801 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f25a9678900 15 stack 0x5636c4d32ac0 end 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is done 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cef030:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f25a9678900 15 stack 0x5636c4cd5900 end 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is done 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f25a9678900 4 starting trim on bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4882a00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4882a00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f25a9678900 20 enqueued request req=0x5636c4883ca0 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f25a9678900 20 req: 0x5636c4883ca0 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is io blocked 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f255b7ee640 20 dequeued request req=0x5636c4883ca0 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f255b7ee640 20 RGWWQ: empty 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f255b7ee640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 : miss 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f255b7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f255b7ee640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f255b7ee640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 info.flags=0x17 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f255b7ee640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 to cache LRU end 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f255b7ee640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f255b7ee640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f255b7ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f255b7ee640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-26 : miss 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f255b7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f255b7ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f255b7ee640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-26 info.flags=0x0 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f255b7ee640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-26 to cache LRU end 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f255b7ee640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-26 : miss 2026-03-20T11:46:48.802 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f255b7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f255b7ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f255b7ee640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-26 info.flags=0x0 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f255b7ee640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-26 to cache LRU end 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.805+0000 7f255b7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31): adding source pipe: {s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f255b7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31): adding dest pipe: {s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f255b7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f255b7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f255b7ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d3e7a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d3e7a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4882a00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d44500:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 15 stack 0x5636c4ce56a0 end 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 run: stack=0x5636c4ce56a0 is done 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4882a00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 collect(): s=0x5636c4d3c450 stack=0x5636c4ce56a0 is complete 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 15 stack 0x5636c4d3c450 end 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 stack->operate() returned ret=-22 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 run: stack=0x5636c4d3c450 is done 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4882a00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d39990:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 15 stack 0x5636c4873b00 end 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 run: stack=0x5636c4873b00 is done 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4882a00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4ccd0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.803 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 collect(): s=0x5636c4cdf880 stack=0x5636c4873b00 is complete 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-22[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27]) markers=[,,,,,,,,00000000005.29.12,,], shards=11 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 4 starting trim on bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4cef030:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 10 trimming bilog shard 8 of :dipepu-22[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27]) at marker 00000000005.29.12 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d53750:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4ce5ff0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 enqueued request req=0x5636c4ce4c50 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 req: 0x5636c4ce4c50 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 run: stack=0x5636c4d3c450 is io blocked 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d41e30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4ce5ff0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27.8 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 run: stack=0x5636c4ce56a0 is io blocked 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d41e30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f255afed640 20 dequeued request req=0x5636c4ce4c50 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f255afed640 20 RGWWQ: empty 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f255afed640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 : miss 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f255afed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 0964301c3993956d3c4c46675e30e3c4f09f508963abb8138320f5cda0b5bb88 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:0964301c3993956d3c4c46675e30e3c4f09f508963abb8138320f5cda0b5bb88 2026-03-20T11:46:48.804 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 96068b52261898a98cf7a2b909d1d7535cccfba2406fed81ca3ae767f6712323 2026-03-20T11:46:48.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=96068b52261898a98cf7a2b909d1d7535cccfba2406fed81ca3ae767f6712323 2026-03-20T11:46:48.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=21, curl_handle=0x5636c4ce15f0 2026-03-20T11:46:48.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f2506ffd640 20 link_request req_data=0x5636c4886970 req_data->id=21, curl_handle=0x5636c4ce15f0 2026-03-20T11:46:48.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f25a9678900 20 run: stack=0x5636c4873b00 is io blocked 2026-03-20T11:46:48.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f255afed640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f255afed640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 info.flags=0x17 2026-03-20T11:46:48.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f255afed640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 to cache LRU end 2026-03-20T11:46:48.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f255afed640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.805 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f255afed640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.805 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:48.806+0000 7f12deaad640 -1 req 426961992053273037 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.806+0000 7f255afed640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f255afed640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-51 : miss 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f255afed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f255afed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f255afed640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-51 info.flags=0x0 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f255afed640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-51 to cache LRU end 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f255afed640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-51 : miss 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f255afed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f255afed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f255afed640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-51 info.flags=0x0 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f255afed640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-51 to cache LRU end 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f255afed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11): adding source pipe: {s={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f255afed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11): adding dest pipe: {s={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f255afed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f255afed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f255afed640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d39990:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 run: stack=0x5636c4d3c450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 84bf724bba4c40e872b7f92567ef4b01527d391051af60d697d3ab49c258ea5a 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:84bf724bba4c40e872b7f92567ef4b01527d391051af60d697d3ab49c258ea5a 2026-03-20T11:46:48.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = ce84cd23000c008dbf798c84ed208ef2255664e02e95b98093006bd918304165 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=ce84cd23000c008dbf798c84ed208ef2255664e02e95b98093006bd918304165 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=22, curl_handle=0x5636c4cdd120 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f2506ffd640 20 link_request req_data=0x5636c4ce1d00 req_data->id=22, curl_handle=0x5636c4cdd120 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is io blocked 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4ce5ff0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4ce5ff0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 15 stack 0x5636c4ce56a0 end 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 run: stack=0x5636c4ce56a0 is done 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4cef030:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4ccd0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 15 stack 0x5636c4cdf880 end 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is done 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d45160:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 4 starting trim on bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 enqueued request req=0x5636c4cce570 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 req: 0x5636c4cce570 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.807+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is io blocked 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.808+0000 7f255a7ec640 20 dequeued request req=0x5636c4cce570 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.808+0000 7f255a7ec640 20 RGWWQ: empty 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f255a7ec640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 : miss 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f255a7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f255a7ec640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f255a7ec640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 info.flags=0x17 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f255a7ec640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 to cache LRU end 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f255a7ec640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f255a7ec640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f255a7ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f255a7ec640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-55 : miss 2026-03-20T11:46:48.807 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f255a7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f255a7ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f255a7ec640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-55 info.flags=0x0 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f255a7ec640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-55 to cache LRU end 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f255a7ec640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-55 : miss 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f255a7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f255a7ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f255a7ec640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-55 info.flags=0x0 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f255a7ec640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-55 to cache LRU end 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f255a7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding source pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f255a7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding dest pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f255a7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f255a7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f255a7ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d45160:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d4e950:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d45160:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d44500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d44500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 439e87d5d49679b4505f10e7ae15a9e21a8fba1af324cf5ff7fca749d4c96fdd 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:439e87d5d49679b4505f10e7ae15a9e21a8fba1af324cf5ff7fca749d4c96fdd 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.808 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.809 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.809 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 63c3cabd760e8e78a99f7ac6cfee7600cbbed6fb5e5bf67c28f06be5b9ce035e 2026-03-20T11:46:48.809 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=63c3cabd760e8e78a99f7ac6cfee7600cbbed6fb5e5bf67c28f06be5b9ce035e 2026-03-20T11:46:48.809 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.809 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.809 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.809 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=23, curl_handle=0x5636c4cd9c00 2026-03-20T11:46:48.809 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f2506ffd640 20 link_request req_data=0x5636c4cda310 req_data->id=23, curl_handle=0x5636c4cd9c00 2026-03-20T11:46:48.809 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.809+0000 7f25a9678900 20 run: stack=0x5636c4ce56a0 is io blocked 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cebca0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f25a9678900 15 stack 0x5636c4884340 end 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f25a9678900 20 run: stack=0x5636c4884340 is done 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f25a9678900 20 collect(): s=0x5636c4cce390 stack=0x5636c4884340 is complete 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f25a9678900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f25a9678900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f25a9678900 15 stack 0x5636c4cce390 end 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f25a9678900 20 stack->operate() returned ret=-22 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is done 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f25a9678900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f25a9678900 4 starting trim on bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f25a9678900 20 enqueued request req=0x5636c4882640 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f25a9678900 20 req: 0x5636c4882640 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is io blocked 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f2559feb640 20 dequeued request req=0x5636c4882640 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f2559feb640 20 RGWWQ: empty 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f2559feb640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 : miss 2026-03-20T11:46:48.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f2559feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.837 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f2559feb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.837 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f2559feb640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 info.flags=0x17 2026-03-20T11:46:48.837 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f2559feb640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 to cache LRU end 2026-03-20T11:46:48.837 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f2559feb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.837 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f2559feb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:48.837 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f2559feb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:48.837 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f2559feb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-28 : miss 2026-03-20T11:46:48.837 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.839+0000 7f2559feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.837 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f2559feb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.837 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f2559feb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-28 info.flags=0x0 2026-03-20T11:46:48.837 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f2559feb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-28 to cache LRU end 2026-03-20T11:46:48.837 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f2559feb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-28 : miss 2026-03-20T11:46:48.837 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f2559feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f2559feb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f2559feb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-28 info.flags=0x0 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f2559feb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-28 to cache LRU end 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f2559feb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding source pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f2559feb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding dest pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f2559feb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f2559feb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f2559feb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cebca0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.838 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.839 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.839 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.839 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.839 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.839 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.839 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 559eaafdf2adcdcd4909b428ffe2a8942212899ce4613522a2a1e5def2fb1c98 2026-03-20T11:46:48.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.839 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.839 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.839 INFO:teuthology.orchestra.run.vm02.stderr:559eaafdf2adcdcd4909b428ffe2a8942212899ce4613522a2a1e5def2fb1c98 2026-03-20T11:46:48.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = c2198fcd2d8add6121d4ac97266f7869f6405f8aa04bdc2477357de26e4b9d15 2026-03-20T11:46:48.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c2198fcd2d8add6121d4ac97266f7869f6405f8aa04bdc2477357de26e4b9d15 2026-03-20T11:46:48.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=24, curl_handle=0x5636c4884b90 2026-03-20T11:46:48.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f2506ffd640 20 link_request req_data=0x5636c4cdd870 req_data->id=24, curl_handle=0x5636c4884b90 2026-03-20T11:46:48.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.840+0000 7f25a9678900 20 run: stack=0x5636c4884340 is io blocked 2026-03-20T11:46:48.839 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:48.841+0000 7f128c208640 -1 req 13681611513541993608 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d41e30:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d41e30:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d53750:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25a9678900 15 stack 0x5636c4873b00 end 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25a9678900 20 run: stack=0x5636c4873b00 is done 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25a9678900 20 collect(): s=0x5636c4cd5900 stack=0x5636c4873b00 is complete 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25a9678900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25a9678900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25a9678900 15 stack 0x5636c4cd5900 end 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25a9678900 20 stack->operate() returned ret=-22 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is done 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25a9678900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25a9678900 4 starting trim on bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25a9678900 20 enqueued request req=0x5636c4ce5e00 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25a9678900 20 req: 0x5636c4ce5e00 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is io blocked 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25597ea640 20 dequeued request req=0x5636c4ce5e00 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25597ea640 20 RGWWQ: empty 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25597ea640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 : miss 2026-03-20T11:46:48.844 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25597ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25597ea640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25597ea640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 info.flags=0x17 2026-03-20T11:46:48.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25597ea640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 to cache LRU end 2026-03-20T11:46:48.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25597ea640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25597ea640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:48.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25597ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:48.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25597ea640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-31 : miss 2026-03-20T11:46:48.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.847+0000 7f25597ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25597ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25597ea640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-31 info.flags=0x0 2026-03-20T11:46:48.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25597ea640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-31 to cache LRU end 2026-03-20T11:46:48.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25597ea640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-31 : miss 2026-03-20T11:46:48.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25597ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25597ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25597ea640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-31 info.flags=0x0 2026-03-20T11:46:48.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25597ea640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-31 to cache LRU end 2026-03-20T11:46:48.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25597ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding source pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25597ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding dest pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25597ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25597ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25597ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d53750:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4cef030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4cef030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = a9ff993b3cc3ad2e323dcd38e1407e5f02a77415d07ad2e7015569a3d03d4de7 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:a9ff993b3cc3ad2e323dcd38e1407e5f02a77415d07ad2e7015569a3d03d4de7 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = e4c36062857a9bc6749007790b6f5ecb90dc03a06162b6d831a34abcf6dbb75c 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=e4c36062857a9bc6749007790b6f5ecb90dc03a06162b6d831a34abcf6dbb75c 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=25, curl_handle=0x5636c4ce15f0 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f2506ffd640 20 link_request req_data=0x5636c4886970 req_data->id=25, curl_handle=0x5636c4ce15f0 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.848+0000 7f25a9678900 20 run: stack=0x5636c4873b00 is io blocked 2026-03-20T11:46:48.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.846 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:48.849+0000 7f123c969640 -1 req 11394076341905050625 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d39990:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 15 stack 0x5636c4d32ac0 end 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is done 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 20 collect(): s=0x5636c4d3c450 stack=0x5636c4d32ac0 is complete 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) markers=[00000000002.62.6,,00000000002.48.6,,00000000002.70.6,,,00000000002.59.6,,,], shards=11 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd9cb0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 10 trimming bilog shard 0 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.62.6 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 10 trimming bilog shard 2 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.48.6 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 10 trimming bilog shard 4 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.70.6 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 10 trimming bilog shard 7 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.59.6 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 20 run: stack=0x5636c4d3c450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4ce37b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4ce5ff0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4d50d00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4875160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4ce37b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.0 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is io blocked 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4ce5ff0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.2 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is io blocked 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4d50d00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.4 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 20 run: stack=0x5636c48737c0 is io blocked 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4875160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.7 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.849+0000 7f25a9678900 20 run: stack=0x5636c4cee790 is io blocked 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.850+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4ce37b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.850+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4ce37b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.847 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.850+0000 7f25a9678900 15 stack 0x5636c4d32ac0 end 2026-03-20T11:46:48.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.850+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.850+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is done 2026-03-20T11:46:48.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.850+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd9cb0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.850+0000 7f25a9678900 20 run: stack=0x5636c4d3c450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.850+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4ce5ff0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.850+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4ce5ff0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.850+0000 7f25a9678900 15 stack 0x5636c4d3ab70 end 2026-03-20T11:46:48.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.850+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.850+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is done 2026-03-20T11:46:48.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.850+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd9cb0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.850+0000 7f25a9678900 20 run: stack=0x5636c4d3c450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4d50d00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4d50d00:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f25a9678900 15 stack 0x5636c48737c0 end 2026-03-20T11:46:48.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f25a9678900 20 run: stack=0x5636c48737c0 is done 2026-03-20T11:46:48.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd9cb0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f25a9678900 20 run: stack=0x5636c4d3c450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4875160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4875160:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f25a9678900 15 stack 0x5636c4cee790 end 2026-03-20T11:46:48.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f25a9678900 20 run: stack=0x5636c4cee790 is done 2026-03-20T11:46:48.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd9cb0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.848 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:48.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f25a9678900 15 stack 0x5636c4d3c450 end 2026-03-20T11:46:48.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f25a9678900 20 run: stack=0x5636c4d3c450 is done 2026-03-20T11:46:48.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f25a9678900 4 starting trim on bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:48.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f25a9678900 20 enqueued request req=0x5636c4cd0430 2026-03-20T11:46:48.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f25a9678900 20 req: 0x5636c4cd0430 2026-03-20T11:46:48.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f25a9678900 20 run: stack=0x5636c4d3c450 is io blocked 2026-03-20T11:46:48.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f2558fe9640 20 dequeued request req=0x5636c4cd0430 2026-03-20T11:46:48.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f2558fe9640 20 RGWWQ: empty 2026-03-20T11:46:48.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f2558fe9640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 : miss 2026-03-20T11:46:48.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f2558fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f2558fe9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f2558fe9640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 info.flags=0x17 2026-03-20T11:46:48.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f2558fe9640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 to cache LRU end 2026-03-20T11:46:48.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f2558fe9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f2558fe9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f2558fe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:48.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f2558fe9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-39 : miss 2026-03-20T11:46:48.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.851+0000 7f2558fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d44500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.849 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f2558fe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f2558fe9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-39 info.flags=0x0 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f2558fe9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-39 to cache LRU end 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f2558fe9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-39 : miss 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f2558fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d44500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d44500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d44500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d4e950:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 15 stack 0x5636c4ce56a0 end 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 run: stack=0x5636c4ce56a0 is done 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d45160:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 collect(): s=0x5636c4cdf880 stack=0x5636c4ce56a0 is complete 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-55[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15]) markers=[,,,,,00000000001.133.6,,,,,], shards=11 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4cd9cb0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 10 trimming bilog shard 5 of :dipepu-55[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15]) at marker 00000000001.133.6 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d41c70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d41c70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15.5 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 run: stack=0x5636c4ce56a0 is io blocked 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f2558fe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f2558fe9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-39 info.flags=0x0 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f2558fe9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-39 to cache LRU end 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f2558fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding source pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f2558fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding dest pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f2558fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f2558fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f2558fe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d39990:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 run: stack=0x5636c4d3c450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.850 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 2a3126eaa769bf5086a9aa1ef9949f6ae4a0d4ff318920c95deb2655940921b0 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2a3126eaa769bf5086a9aa1ef9949f6ae4a0d4ff318920c95deb2655940921b0 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = dd590a4433cebe08602209324c32da1ae204bc56816ff1cf4c60439d5e3c8098 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=dd590a4433cebe08602209324c32da1ae204bc56816ff1cf4c60439d5e3c8098 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=26, curl_handle=0x5636c4cd9c00 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f2506ffd640 20 link_request req_data=0x5636c4cda310 req_data->id=26, curl_handle=0x5636c4cd9c00 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.852+0000 7f25a9678900 20 run: stack=0x5636c4cee790 is io blocked 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.854+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d41c70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.854+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d41c70:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.854+0000 7f25a9678900 15 stack 0x5636c4ce56a0 end 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.854+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.854+0000 7f25a9678900 20 run: stack=0x5636c4ce56a0 is done 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.854+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4cd9cb0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.854+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d45160:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.854+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.854+0000 7f25a9678900 15 stack 0x5636c4cdf880 end 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.854+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is done 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.854+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.854+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.854+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.854+0000 7f25a9678900 4 starting trim on bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.854+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.854+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.854+0000 7f25a9678900 20 enqueued request req=0x5636c4cce570 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.854+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.854+0000 7f25a9678900 20 req: 0x5636c4cce570 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.854+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is io blocked 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.854+0000 7f25587e8640 20 dequeued request req=0x5636c4cce570 2026-03-20T11:46:48.851 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.854+0000 7f25587e8640 20 RGWWQ: empty 2026-03-20T11:46:48.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.854+0000 7f25587e8640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 : miss 2026-03-20T11:46:48.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.854+0000 7f25587e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25587e8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:46:48.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25587e8640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 info.flags=0x17 2026-03-20T11:46:48.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25587e8640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 to cache LRU end 2026-03-20T11:46:48.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25587e8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25587e8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25587e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:48.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25587e8640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-8 : miss 2026-03-20T11:46:48.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25587e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25587e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25587e8640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-8 info.flags=0x0 2026-03-20T11:46:48.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25587e8640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-8 to cache LRU end 2026-03-20T11:46:48.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25587e8640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-8 : miss 2026-03-20T11:46:48.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25587e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25587e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25587e8640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-8 info.flags=0x0 2026-03-20T11:46:48.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25587e8640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-8 to cache LRU end 2026-03-20T11:46:48.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25587e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding source pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25587e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding dest pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25587e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25587e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25587e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:48.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d44500:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d507f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d507f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = e657de060ecea3863a950d9b770021f34db7a2eb22155311926bd07fe81cbc31 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:e657de060ecea3863a950d9b770021f34db7a2eb22155311926bd07fe81cbc31 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = f4b7d05152475b951075353bdf9c003eacce99755b0d1e2c870ac9dbc255588e 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=f4b7d05152475b951075353bdf9c003eacce99755b0d1e2c870ac9dbc255588e 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=27, curl_handle=0x5636c4cdd120 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f2506ffd640 20 link_request req_data=0x5636c4ce1d00 req_data->id=27, curl_handle=0x5636c4cdd120 2026-03-20T11:46:48.853 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.855+0000 7f25a9678900 20 run: stack=0x5636c4ce56a0 is io blocked 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cebca0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f25a9678900 15 stack 0x5636c4884340 end 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f25a9678900 20 run: stack=0x5636c4884340 is done 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f25a9678900 20 collect(): s=0x5636c4cce390 stack=0x5636c4884340 is complete 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f25a9678900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f25a9678900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f25a9678900 15 stack 0x5636c4cce390 end 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f25a9678900 20 stack->operate() returned ret=-22 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is done 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f25a9678900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f25a9678900 4 starting trim on bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f25a9678900 20 enqueued request req=0x5636c4882640 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f25a9678900 20 req: 0x5636c4882640 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is io blocked 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f2557fe7640 20 dequeued request req=0x5636c4882640 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f2557fe7640 20 RGWWQ: empty 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f2557fe7640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 : miss 2026-03-20T11:46:48.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f2557fe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f2557fe7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f2557fe7640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 info.flags=0x17 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f2557fe7640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 to cache LRU end 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f2557fe7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f2557fe7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f2557fe7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f2557fe7640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-16 : miss 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.882+0000 7f2557fe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f2557fe7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f2557fe7640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-16 info.flags=0x0 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f2557fe7640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-16 to cache LRU end 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f2557fe7640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-16 : miss 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f2557fe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f2557fe7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f2557fe7640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-16 info.flags=0x0 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f2557fe7640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-16 to cache LRU end 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f2557fe7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding source pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f2557fe7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding dest pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f2557fe7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f2557fe7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f2557fe7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cebca0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 6d1746dd1cc1f72b6325538556c57d07493a5f6a6d3d48439071091b8f25456b 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:6d1746dd1cc1f72b6325538556c57d07493a5f6a6d3d48439071091b8f25456b 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 6d366f9d03c2ef426a0282951e7a5f018225890d1f75ec1b707eed8f7761a94a 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=6d366f9d03c2ef426a0282951e7a5f018225890d1f75ec1b707eed8f7761a94a 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=28, curl_handle=0x5636c4884b90 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f2506ffd640 20 link_request req_data=0x5636c4cdd870 req_data->id=28, curl_handle=0x5636c4884b90 2026-03-20T11:46:48.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.883+0000 7f25a9678900 20 run: stack=0x5636c4884340 is io blocked 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4cef030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4cef030:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d53750:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25a9678900 15 stack 0x5636c4873b00 end 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25a9678900 20 run: stack=0x5636c4873b00 is done 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25a9678900 20 collect(): s=0x5636c4cd5900 stack=0x5636c4873b00 is complete 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25a9678900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25a9678900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25a9678900 15 stack 0x5636c4cd5900 end 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25a9678900 20 stack->operate() returned ret=-22 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is done 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25a9678900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25a9678900 4 starting trim on bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25a9678900 20 enqueued request req=0x5636c4ce5e00 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25a9678900 20 req: 0x5636c4ce5e00 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is io blocked 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25577e6640 20 dequeued request req=0x5636c4ce5e00 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25577e6640 20 RGWWQ: empty 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25577e6640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 : miss 2026-03-20T11:46:48.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25577e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25577e6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25577e6640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 info.flags=0x17 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25577e6640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 to cache LRU end 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25577e6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25577e6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25577e6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25577e6640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-43 : miss 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25577e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25577e6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25577e6640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-43 info.flags=0x0 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25577e6640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-43 to cache LRU end 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25577e6640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-43 : miss 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.890+0000 7f25577e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25577e6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25577e6640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-43 info.flags=0x0 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25577e6640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-43 to cache LRU end 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25577e6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding source pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25577e6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding dest pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25577e6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25577e6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25577e6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d37e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4cef030:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d50e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d50e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.888 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.889 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.889 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.889 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.889 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.889 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.889 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.889 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 2841bafdf20a53585b2e6e8621e70ec5686baf9027f5a33f874a978b4b6440c6 2026-03-20T11:46:48.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.889 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.889 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.889 INFO:teuthology.orchestra.run.vm02.stderr:2841bafdf20a53585b2e6e8621e70ec5686baf9027f5a33f874a978b4b6440c6 2026-03-20T11:46:48.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = a3a93989d51367afc17a5fde8b16ef69d2ab5a7a11c49b20dd28fa15e74df47b 2026-03-20T11:46:48.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a3a93989d51367afc17a5fde8b16ef69d2ab5a7a11c49b20dd28fa15e74df47b 2026-03-20T11:46:48.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=29, curl_handle=0x5636c4ce15f0 2026-03-20T11:46:48.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f2506ffd640 20 link_request req_data=0x5636c4886970 req_data->id=29, curl_handle=0x5636c4ce15f0 2026-03-20T11:46:48.889 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:48.891+0000 7f1291a13640 -1 req 7316644150462391819 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:48.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.891+0000 7f25a9678900 20 run: stack=0x5636c4873b00 is io blocked 2026-03-20T11:46:48.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.895+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.895+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.895+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.895+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.895+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d39990:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.895+0000 7f25a9678900 15 stack 0x5636c4cee790 end 2026-03-20T11:46:48.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.895+0000 7f25a9678900 20 run: stack=0x5636c4cee790 is done 2026-03-20T11:46:48.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.895+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.895+0000 7f25a9678900 20 collect(): s=0x5636c4d3c450 stack=0x5636c4cee790 is complete 2026-03-20T11:46:48.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.895+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.895+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-39[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44]) markers=[,,,,,,,,00000000001.18.6,,], shards=11 2026-03-20T11:46:48.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.895+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d39990:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.895+0000 7f25a9678900 10 trimming bilog shard 8 of :dipepu-39[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44]) at marker 00000000001.18.6 2026-03-20T11:46:48.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.895+0000 7f25a9678900 20 run: stack=0x5636c4d3c450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.895+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d4e300:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.895+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d4e300:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.895+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44.8 2026-03-20T11:46:48.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.895+0000 7f25a9678900 20 run: stack=0x5636c4cee790 is io blocked 2026-03-20T11:46:48.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.896+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d4e300:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.896+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d4e300:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.896+0000 7f25a9678900 15 stack 0x5636c4cee790 end 2026-03-20T11:46:48.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.896+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.896+0000 7f25a9678900 20 run: stack=0x5636c4cee790 is done 2026-03-20T11:46:48.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.896+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d39990:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.896+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.896+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:48.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.896+0000 7f25a9678900 15 stack 0x5636c4d3c450 end 2026-03-20T11:46:48.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.896+0000 7f25a9678900 20 run: stack=0x5636c4d3c450 is done 2026-03-20T11:46:48.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.896+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.896+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.896+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.896+0000 7f25a9678900 4 starting trim on bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:48.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.896+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d47220:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.896+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d47220:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.896+0000 7f25a9678900 20 enqueued request req=0x5636c4cd0430 2026-03-20T11:46:48.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.896+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.896+0000 7f25a9678900 20 req: 0x5636c4cd0430 2026-03-20T11:46:48.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.896+0000 7f25a9678900 20 run: stack=0x5636c4d3c450 is io blocked 2026-03-20T11:46:48.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.896+0000 7f256effd640 20 dequeued request req=0x5636c4cd0430 2026-03-20T11:46:48.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.896+0000 7f256effd640 20 RGWWQ: empty 2026-03-20T11:46:48.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.896+0000 7f256effd640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 : miss 2026-03-20T11:46:48.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.896+0000 7f256effd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f256effd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f256effd640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 info.flags=0x17 2026-03-20T11:46:48.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f256effd640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 to cache LRU end 2026-03-20T11:46:48.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f256effd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f256effd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f256effd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:48.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f256effd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-10 : miss 2026-03-20T11:46:48.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f256effd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f256effd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f256effd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-10 info.flags=0x0 2026-03-20T11:46:48.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f256effd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-10 to cache LRU end 2026-03-20T11:46:48.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f256effd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-10 : miss 2026-03-20T11:46:48.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f256effd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f256effd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f256effd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-10 info.flags=0x0 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f256effd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-10 to cache LRU end 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f256effd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding source pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f256effd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding dest pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f256effd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f256effd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f256effd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d47220:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d47220:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d47220:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d47220:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d39990:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f25a9678900 20 run: stack=0x5636c4d3c450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.897+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 4ce510c1e83c868a2d6d49d166323f37117c1caf713c26fd3ab64950f38beb8a 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:4ce510c1e83c868a2d6d49d166323f37117c1caf713c26fd3ab64950f38beb8a 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 0d3b3d04e1821fcdc7ef70b31e52acb6d4ad98a0a083d41078296833184f3fff 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=0d3b3d04e1821fcdc7ef70b31e52acb6d4ad98a0a083d41078296833184f3fff 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=30, curl_handle=0x5636c4cdd120 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f2506ffd640 20 link_request req_data=0x5636c4cda310 req_data->id=30, curl_handle=0x5636c4cdd120 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 run: stack=0x5636c4cee790 is io blocked 2026-03-20T11:46:48.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d507f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d507f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d507f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d507f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d44500:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 15 stack 0x5636c4ce56a0 end 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 run: stack=0x5636c4ce56a0 is done 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 collect(): s=0x5636c4cdf880 stack=0x5636c4ce56a0 is complete 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) markers=[,00000000002.13.6,,00000000001.5.6,,00000000001.2.6,00000000002.13.6,00000000001.11.6,00000000001.4.6,00000000001.12.6,], shards=11 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d44500:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 10 trimming bilog shard 1 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000002.13.6 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 10 trimming bilog shard 3 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.5.6 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 10 trimming bilog shard 5 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.2.6 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 10 trimming bilog shard 6 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000002.13.6 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 10 trimming bilog shard 7 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.11.6 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 10 trimming bilog shard 8 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.4.6 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 10 trimming bilog shard 9 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.12.6 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4ccd5b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4d41c70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d49d90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d55140:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 cr:s=0x5636c4873f10:op=0x5636c4d55f30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4d504f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4d42750:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4ccd5b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.1 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 run: stack=0x5636c4ce56a0 is io blocked 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4d41c70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.3 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 run: stack=0x5636c48737c0 is io blocked 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d49d90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.5 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is io blocked 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d55140:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.6 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is io blocked 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 cr:s=0x5636c4873f10:op=0x5636c4d55f30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.7 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 run: stack=0x5636c4873f10 is io blocked 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4d504f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.8 2026-03-20T11:46:48.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 run: stack=0x5636c4886580 is io blocked 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4d42750:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.9 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.898+0000 7f25a9678900 20 run: stack=0x5636c4d3f5a0 is io blocked 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4d41c70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4d41c70:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 15 stack 0x5636c48737c0 end 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 run: stack=0x5636c48737c0 is done 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d44500:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d49d90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d49d90:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 15 stack 0x5636c4d3ab70 end 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is done 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d44500:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4ccd5b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4ccd5b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 15 stack 0x5636c4ce56a0 end 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 run: stack=0x5636c4ce56a0 is done 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4d42750:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4d42750:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 15 stack 0x5636c4d3f5a0 end 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 run: stack=0x5636c4d3f5a0 is done 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 cr:s=0x5636c4873f10:op=0x5636c4d55f30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 cr:s=0x5636c4873f10:op=0x5636c4d55f30:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 15 stack 0x5636c4873f10 end 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 run: stack=0x5636c4873f10 is done 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4d504f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4d504f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 15 stack 0x5636c4886580 end 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 run: stack=0x5636c4886580 is done 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d44500:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d44500:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d44500:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d44500:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.900+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d55140:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d55140:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f25a9678900 15 stack 0x5636c4d32ac0 end 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is done 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d44500:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f25a9678900 20 cr:s=0x5636c4cdf880:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f25a9678900 15 stack 0x5636c4cdf880 end 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f25a9678900 20 run: stack=0x5636c4cdf880 is done 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f25a9678900 4 starting trim on bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d4af70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d4af70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f25a9678900 20 enqueued request req=0x5636c4cd9200 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f25a9678900 20 req: 0x5636c4cd9200 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is io blocked 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f256e7fc640 20 dequeued request req=0x5636c4cd9200 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f256e7fc640 20 RGWWQ: empty 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f256e7fc640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 : miss 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f256e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f256e7fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f256e7fc640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 info.flags=0x17 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f256e7fc640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 to cache LRU end 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f256e7fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f256e7fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f256e7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f256e7fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-3 : miss 2026-03-20T11:46:48.898 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f256e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f256e7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f256e7fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-3 info.flags=0x0 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f256e7fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-3 to cache LRU end 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f256e7fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-3 : miss 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f256e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f256e7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f256e7fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-3 info.flags=0x0 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f256e7fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-3 to cache LRU end 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f256e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding source pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.901+0000 7f256e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding dest pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f256e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f256e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f256e7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d4af70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d4af70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d4af70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d4af70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4cdfbe0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4d44500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4d44500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.899 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.900 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.900 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.900 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 75626962b03a42e593e72dc5cc5bd4af4478a2c2c56327cc793494053243e07c 2026-03-20T11:46:48.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.900 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.900 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.900 INFO:teuthology.orchestra.run.vm02.stderr:75626962b03a42e593e72dc5cc5bd4af4478a2c2c56327cc793494053243e07c 2026-03-20T11:46:48.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = c95c11e3cfbd43903be9fe632becb1ec227390ee9575505fe6464b3f29a90f29 2026-03-20T11:46:48.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c95c11e3cfbd43903be9fe632becb1ec227390ee9575505fe6464b3f29a90f29 2026-03-20T11:46:48.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=31, curl_handle=0x5636c4cd9c00 2026-03-20T11:46:48.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f2506ffd640 20 link_request req_data=0x5636c4ce1d00 req_data->id=31, curl_handle=0x5636c4cd9c00 2026-03-20T11:46:48.900 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.902+0000 7f25a9678900 20 run: stack=0x5636c4d3f5a0 is io blocked 2026-03-20T11:46:48.900 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:48.902+0000 7f123b967640 -1 req 4962280813439420334 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:48.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.925+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.925+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.925+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.925+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.925+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cebca0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.925+0000 7f25a9678900 15 stack 0x5636c4884340 end 2026-03-20T11:46:48.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.925+0000 7f25a9678900 20 run: stack=0x5636c4884340 is done 2026-03-20T11:46:48.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.925+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.925+0000 7f25a9678900 20 collect(): s=0x5636c4cce390 stack=0x5636c4884340 is complete 2026-03-20T11:46:48.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.925+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.925+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) markers=[,,,,,00000000002.22.11,,,00000000031.119.12,,], shards=11 2026-03-20T11:46:48.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.925+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4ccdab0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.925+0000 7f25a9678900 10 trimming bilog shard 5 of :dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) at marker 00000000002.22.11 2026-03-20T11:46:48.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.925+0000 7f25a9678900 10 trimming bilog shard 8 of :dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) at marker 00000000031.119.12 2026-03-20T11:46:48.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.925+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.925+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4874340:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.925+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4d3acb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.925+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4874340:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.925+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21.5 2026-03-20T11:46:48.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.925+0000 7f25a9678900 20 run: stack=0x5636c4884340 is io blocked 2026-03-20T11:46:48.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.925+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4d3acb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.925+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21.8 2026-03-20T11:46:48.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.925+0000 7f25a9678900 20 run: stack=0x5636c4886580 is io blocked 2026-03-20T11:46:48.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4874340:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4874340:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f25a9678900 15 stack 0x5636c4884340 end 2026-03-20T11:46:48.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f25a9678900 20 run: stack=0x5636c4884340 is done 2026-03-20T11:46:48.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4ccdab0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4d3acb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4d3acb0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f25a9678900 15 stack 0x5636c4886580 end 2026-03-20T11:46:48.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f25a9678900 20 run: stack=0x5636c4886580 is done 2026-03-20T11:46:48.924 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4ccdab0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.924 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.924 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:48.924 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f25a9678900 15 stack 0x5636c4cce390 end 2026-03-20T11:46:48.924 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is done 2026-03-20T11:46:48.924 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.924 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.924 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.924 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f25a9678900 4 starting trim on bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:48.924 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d4af70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.924 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d4af70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.924 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f25a9678900 20 enqueued request req=0x5636c4cd9410 2026-03-20T11:46:48.924 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.924 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f25a9678900 20 req: 0x5636c4cd9410 2026-03-20T11:46:48.924 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is io blocked 2026-03-20T11:46:48.924 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f256dffb640 20 dequeued request req=0x5636c4cd9410 2026-03-20T11:46:48.924 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f256dffb640 20 RGWWQ: empty 2026-03-20T11:46:48.924 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.926+0000 7f256dffb640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 : miss 2026-03-20T11:46:48.924 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f256dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f256dffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f256dffb640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 info.flags=0x17 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f256dffb640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 to cache LRU end 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f256dffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f256dffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f256dffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f256dffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-9 : miss 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f256dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f256dffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f256dffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-9 info.flags=0x0 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f256dffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-9 to cache LRU end 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f256dffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-9 : miss 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f256dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f256dffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f256dffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-9 info.flags=0x0 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f256dffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-9 to cache LRU end 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f256dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding source pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f256dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding dest pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f256dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f256dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f256dffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d4af70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d4af70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d4af70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d4af70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4cdd140:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4d546f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4d546f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.927+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.928+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.928+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.928+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 5b48c51556648f04774d279a8c224eb3a7e371cf12b4c3c6638e9466f64de67f 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.928+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:5b48c51556648f04774d279a8c224eb3a7e371cf12b4c3c6638e9466f64de67f 2026-03-20T11:46:48.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.928+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.926 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.928+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.926 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.928+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.926 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.928+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.926 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.928+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 5c78cef609b9cc4e3f78b970d77a2e23bb9a632a9b916af3e50bb2f90a208181 2026-03-20T11:46:48.926 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.928+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5c78cef609b9cc4e3f78b970d77a2e23bb9a632a9b916af3e50bb2f90a208181 2026-03-20T11:46:48.926 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.928+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.926 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.928+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.926 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.928+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.926 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.928+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=32, curl_handle=0x5636c4884b90 2026-03-20T11:46:48.926 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.928+0000 7f2506ffd640 20 link_request req_data=0x5636c4cdd870 req_data->id=32, curl_handle=0x5636c4884b90 2026-03-20T11:46:48.926 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.928+0000 7f25a9678900 20 run: stack=0x5636c4886580 is io blocked 2026-03-20T11:46:48.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d50e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d50e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4cef030:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f25a9678900 15 stack 0x5636c4873b00 end 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f25a9678900 20 run: stack=0x5636c4873b00 is done 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f25a9678900 20 collect(): s=0x5636c4cd5900 stack=0x5636c4873b00 is complete 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f25a9678900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f25a9678900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f25a9678900 15 stack 0x5636c4cd5900 end 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f25a9678900 20 stack->operate() returned ret=-22 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is done 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f25a9678900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f25a9678900 4 starting trim on bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d4af70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d4af70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f25a9678900 20 enqueued request req=0x5636c4ce5e00 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f25a9678900 20 req: 0x5636c4ce5e00 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is io blocked 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f256d7fa640 20 dequeued request req=0x5636c4ce5e00 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f256d7fa640 20 RGWWQ: empty 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f256d7fa640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 : miss 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f256d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f256d7fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f256d7fa640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 info.flags=0x17 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f256d7fa640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 to cache LRU end 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f256d7fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f256d7fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f256d7fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f256d7fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-38 : miss 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f256d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f256d7fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f256d7fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-38 info.flags=0x0 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f256d7fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-38 to cache LRU end 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f256d7fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-38 : miss 2026-03-20T11:46:48.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.932+0000 7f256d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f256d7fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f256d7fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-38 info.flags=0x0 2026-03-20T11:46:48.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f256d7fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-38 to cache LRU end 2026-03-20T11:46:48.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f256d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding source pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f256d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding dest pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f256d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f256d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f256d7fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:48.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d4af70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d4af70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d4af70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d4af70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4cef030:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d50e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d50e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.930 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.930 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.930 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 4f0cd26e9b9300cd3fb961ec78f9c2bce4601c51abd116b2aa5983c9f3c61d7a 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr:4f0cd26e9b9300cd3fb961ec78f9c2bce4601c51abd116b2aa5983c9f3c61d7a 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 27f1581022af433577545601ea0782df31cb2eed603d1c56fd163a656c457d69 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=27f1581022af433577545601ea0782df31cb2eed603d1c56fd163a656c457d69 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=33, curl_handle=0x5636c4ce15f0 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f2506ffd640 20 link_request req_data=0x5636c4886970 req_data->id=33, curl_handle=0x5636c4ce15f0 2026-03-20T11:46:48.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.933+0000 7f25a9678900 20 run: stack=0x5636c4873b00 is io blocked 2026-03-20T11:46:48.937 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.941+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.941+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.941+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.941+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.941+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d39990:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.941+0000 7f25a9678900 15 stack 0x5636c4cee790 end 2026-03-20T11:46:48.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.941+0000 7f25a9678900 20 run: stack=0x5636c4cee790 is done 2026-03-20T11:46:48.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.941+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.941+0000 7f25a9678900 20 collect(): s=0x5636c4d3c450 stack=0x5636c4cee790 is complete 2026-03-20T11:46:48.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.941+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.941+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-10[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15]) markers=[,00000000002.9.6,,,,,,,,,], shards=11 2026-03-20T11:46:48.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.941+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d39990:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.941+0000 7f25a9678900 10 trimming bilog shard 1 of :dipepu-10[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15]) at marker 00000000002.9.6 2026-03-20T11:46:48.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.941+0000 7f25a9678900 20 run: stack=0x5636c4d3c450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.941+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d4c9f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.941+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d4c9f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.941+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15.1 2026-03-20T11:46:48.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.941+0000 7f25a9678900 20 run: stack=0x5636c4cee790 is io blocked 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d4c9f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d4c9f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f25a9678900 15 stack 0x5636c4cee790 end 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f25a9678900 20 run: stack=0x5636c4cee790 is done 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d39990:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f25a9678900 15 stack 0x5636c4d3c450 end 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f25a9678900 20 run: stack=0x5636c4d3c450 is done 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f25a9678900 4 starting trim on bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d51c30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d51c30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f25a9678900 20 enqueued request req=0x5636c4cd0430 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f25a9678900 20 req: 0x5636c4cd0430 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f25a9678900 20 run: stack=0x5636c4d3c450 is io blocked 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f2563fff640 20 dequeued request req=0x5636c4cd0430 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f2563fff640 20 RGWWQ: empty 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f2563fff640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 : miss 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f2563fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f2563fff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f2563fff640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 info.flags=0x17 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f2563fff640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 to cache LRU end 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f2563fff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f2563fff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f2563fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f2563fff640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-15 : miss 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f2563fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f2563fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.939 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f2563fff640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-15 info.flags=0x0 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f2563fff640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-15 to cache LRU end 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f2563fff640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-15 : miss 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.942+0000 7f2563fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f2563fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f2563fff640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-15 info.flags=0x0 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f2563fff640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-15 to cache LRU end 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f2563fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding source pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f2563fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding dest pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f2563fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f2563fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f2563fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d51c30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d51c30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d51c30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d51c30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d39990:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 20 run: stack=0x5636c4d3c450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.940 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.941 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.941 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.941 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.941 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.941 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = bf184bcd0dc5a2aa1859b5055f0c24ae0a82bde51b61ecf511c18668a2381fce 2026-03-20T11:46:48.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.941 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.941 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.941 INFO:teuthology.orchestra.run.vm02.stderr:bf184bcd0dc5a2aa1859b5055f0c24ae0a82bde51b61ecf511c18668a2381fce 2026-03-20T11:46:48.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = bff3906b830791c6c9f20dbe95860b3d2b1ee0a14edce7a7c757542fa6a891b6 2026-03-20T11:46:48.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=bff3906b830791c6c9f20dbe95860b3d2b1ee0a14edce7a7c757542fa6a891b6 2026-03-20T11:46:48.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=34, curl_handle=0x5636c4cdd120 2026-03-20T11:46:48.941 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f2506ffd640 20 link_request req_data=0x5636c4cda310 req_data->id=34, curl_handle=0x5636c4cdd120 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.943+0000 7f25a9678900 20 run: stack=0x5636c4cee790 is io blocked 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4d44500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4d44500:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4cdfbe0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f25a9678900 15 stack 0x5636c4d3f5a0 end 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f25a9678900 20 run: stack=0x5636c4d3f5a0 is done 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f25a9678900 20 collect(): s=0x5636c4d32ac0 stack=0x5636c4d3f5a0 is complete 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f25a9678900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f25a9678900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f25a9678900 15 stack 0x5636c4d32ac0 end 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f25a9678900 20 stack->operate() returned ret=-22 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is done 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f25a9678900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f25a9678900 4 starting trim on bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4cdfbe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4cdfbe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f25a9678900 20 enqueued request req=0x5636c4cd9200 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f25a9678900 20 req: 0x5636c4cd9200 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is io blocked 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f256cff9640 20 dequeued request req=0x5636c4cd9200 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f256cff9640 20 RGWWQ: empty 2026-03-20T11:46:48.942 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f256cff9640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 : miss 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f256cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f256cff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f256cff9640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 info.flags=0x17 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f256cff9640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 to cache LRU end 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f256cff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f256cff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f256cff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f256cff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-13 : miss 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f256cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f256cff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f256cff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-13 info.flags=0x0 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f256cff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-13 to cache LRU end 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f256cff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-13 : miss 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.944+0000 7f256cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f256cff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f256cff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-13 info.flags=0x0 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f256cff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-13 to cache LRU end 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f256cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18): adding source pipe: {s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f256cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18): adding dest pipe: {s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f256cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f256cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f256cff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4cdfbe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4cdfbe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4cdfbe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4cdfbe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4d44500:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4ce9a80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4ce9a80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.943 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.944 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.944 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.944 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.944 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.944 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.944 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.944 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.944 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.944 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 2d3421204dc70cdf4283c5c64871100206634a45094864979a967fc4e0f54db1 2026-03-20T11:46:48.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.944 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.944 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.944 INFO:teuthology.orchestra.run.vm02.stderr:2d3421204dc70cdf4283c5c64871100206634a45094864979a967fc4e0f54db1 2026-03-20T11:46:48.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 2625168834522a8ba635e32190dbdbcc6ae852cff4b0dafab0313ed6d9228844 2026-03-20T11:46:48.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2625168834522a8ba635e32190dbdbcc6ae852cff4b0dafab0313ed6d9228844 2026-03-20T11:46:48.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=35, curl_handle=0x5636c4cd9c00 2026-03-20T11:46:48.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f2506ffd640 20 link_request req_data=0x5636c4ce1d00 req_data->id=35, curl_handle=0x5636c4cd9c00 2026-03-20T11:46:48.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.945+0000 7f25a9678900 20 run: stack=0x5636c4d3f5a0 is io blocked 2026-03-20T11:46:48.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.970+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4d546f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.970+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4d546f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.970+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4d546f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.970+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4d546f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.970+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4cdd140:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.970+0000 7f25a9678900 15 stack 0x5636c4886580 end 2026-03-20T11:46:48.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.970+0000 7f25a9678900 20 run: stack=0x5636c4886580 is done 2026-03-20T11:46:48.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.970+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.970+0000 7f25a9678900 20 collect(): s=0x5636c4cce390 stack=0x5636c4886580 is complete 2026-03-20T11:46:48.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.970+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.970+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-9[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14]) markers=[,00000000002.17.6,,,,,,,,,], shards=11 2026-03-20T11:46:48.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.970+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cdd140:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.970+0000 7f25a9678900 10 trimming bilog shard 1 of :dipepu-9[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14]) at marker 00000000002.17.6 2026-03-20T11:46:48.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.970+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.970+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4d56250:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.970+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4d56250:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.970+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14.1 2026-03-20T11:46:48.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.970+0000 7f25a9678900 20 run: stack=0x5636c4886580 is io blocked 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4d56250:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4d56250:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f25a9678900 15 stack 0x5636c4886580 end 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f25a9678900 20 run: stack=0x5636c4886580 is done 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cdd140:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f25a9678900 15 stack 0x5636c4cce390 end 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is done 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f25a9678900 4 starting trim on bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d3acb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d3acb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f25a9678900 20 enqueued request req=0x5636c4cd9410 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f25a9678900 20 req: 0x5636c4cd9410 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is io blocked 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f2556fe5640 20 dequeued request req=0x5636c4cd9410 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f2556fe5640 20 RGWWQ: empty 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f2556fe5640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 : miss 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f2556fe5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f2556fe5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f2556fe5640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 info.flags=0x17 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f2556fe5640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 to cache LRU end 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f2556fe5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f2556fe5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f2556fe5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:48.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f2556fe5640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-11 : miss 2026-03-20T11:46:48.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.972+0000 7f2556fe5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f2556fe5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f2556fe5640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-11 info.flags=0x0 2026-03-20T11:46:48.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f2556fe5640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-11 to cache LRU end 2026-03-20T11:46:48.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f2556fe5640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-11 : miss 2026-03-20T11:46:48.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f2556fe5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f2556fe5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f2556fe5640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-11 info.flags=0x0 2026-03-20T11:46:48.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f2556fe5640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-11 to cache LRU end 2026-03-20T11:46:48.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f2556fe5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16): adding source pipe: {s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f2556fe5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16): adding dest pipe: {s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f2556fe5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f2556fe5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f2556fe5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:48.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d3acb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d3acb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d3acb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d3acb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4d546f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4ce5ff0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4ce5ff0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 53fff0802b5dc83fae30a747574ae855910a81287a305f674071e16f3f008f2a 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:53fff0802b5dc83fae30a747574ae855910a81287a305f674071e16f3f008f2a 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 072859121b3b4f3a13da7b7b4bf3d5781177f181f44055b85cc5517c44d2a36d 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=072859121b3b4f3a13da7b7b4bf3d5781177f181f44055b85cc5517c44d2a36d 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=36, curl_handle=0x5636c4884b90 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f2506ffd640 20 link_request req_data=0x5636c4cdd870 req_data->id=36, curl_handle=0x5636c4884b90 2026-03-20T11:46:48.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.973+0000 7f25a9678900 20 run: stack=0x5636c4886580 is io blocked 2026-03-20T11:46:48.972 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.976+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d50e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.976+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d50e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.976+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d50e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.976+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d50e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.976+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4cef030:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.976+0000 7f25a9678900 15 stack 0x5636c4873b00 end 2026-03-20T11:46:48.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.976+0000 7f25a9678900 20 run: stack=0x5636c4873b00 is done 2026-03-20T11:46:48.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.976+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.976+0000 7f25a9678900 20 collect(): s=0x5636c4cd5900 stack=0x5636c4873b00 is complete 2026-03-20T11:46:48.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.976+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.976+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-38[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43]) markers=[,,,,,,,,00000000001.55.6,,], shards=11 2026-03-20T11:46:48.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.976+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d4a850:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.976+0000 7f25a9678900 10 trimming bilog shard 8 of :dipepu-38[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43]) at marker 00000000001.55.6 2026-03-20T11:46:48.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.976+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.976+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4ceba10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.976+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4ceba10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.976+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43.8 2026-03-20T11:46:48.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.976+0000 7f25a9678900 20 run: stack=0x5636c4873b00 is io blocked 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4ceba10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4ceba10:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25a9678900 15 stack 0x5636c4873b00 end 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25a9678900 20 run: stack=0x5636c4873b00 is done 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d4a850:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25a9678900 15 stack 0x5636c4cd5900 end 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is done 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25a9678900 4 starting trim on bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d4e2c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d4e2c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25a9678900 20 enqueued request req=0x5636c4ce5e00 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25a9678900 20 req: 0x5636c4ce5e00 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is io blocked 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25627fc640 20 dequeued request req=0x5636c4ce5e00 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25627fc640 20 RGWWQ: empty 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25627fc640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 : miss 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25627fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25627fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25627fc640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 info.flags=0x17 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25627fc640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 to cache LRU end 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25627fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25627fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25627fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25627fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-12 : miss 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.977+0000 7f25627fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25627fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25627fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-12 info.flags=0x0 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25627fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-12 to cache LRU end 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25627fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-12 : miss 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25627fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25627fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25627fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-12 info.flags=0x0 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25627fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-12 to cache LRU end 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25627fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17): adding source pipe: {s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25627fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17): adding dest pipe: {s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25627fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25627fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25627fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d4e2c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d4e2c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d4e2c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d4e2c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4cef030:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d50e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d50e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.975 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 27e2d0fc4468cb6d5910a62e76a2990fc180b68dd082122df15baaf8b0b7ed55 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:27e2d0fc4468cb6d5910a62e76a2990fc180b68dd082122df15baaf8b0b7ed55 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = ecde9de2ceea14d17668150f5e0b365be9012d5733f754bb24b835a28ae927bf 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=ecde9de2ceea14d17668150f5e0b365be9012d5733f754bb24b835a28ae927bf 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=37, curl_handle=0x5636c4ce15f0 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f2506ffd640 20 link_request req_data=0x5636c4886970 req_data->id=37, curl_handle=0x5636c4ce15f0 2026-03-20T11:46:48.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.978+0000 7f25a9678900 20 run: stack=0x5636c4873b00 is io blocked 2026-03-20T11:46:48.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.985+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.985+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.985+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.985+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d38710:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.985+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d39990:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.985+0000 7f25a9678900 15 stack 0x5636c4cee790 end 2026-03-20T11:46:48.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.985+0000 7f25a9678900 20 run: stack=0x5636c4cee790 is done 2026-03-20T11:46:48.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.985+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.985+0000 7f25a9678900 20 collect(): s=0x5636c4d3c450 stack=0x5636c4cee790 is complete 2026-03-20T11:46:48.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.985+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.985+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-15[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20]) markers=[,,,,,,,,00000000007.15.6,,], shards=11 2026-03-20T11:46:48.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.985+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d4a850:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.985+0000 7f25a9678900 10 trimming bilog shard 8 of :dipepu-15[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20]) at marker 00000000007.15.6 2026-03-20T11:46:48.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.985+0000 7f25a9678900 20 run: stack=0x5636c4d3c450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.985+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d51c30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.985+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d51c30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.985+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20.8 2026-03-20T11:46:48.982 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.985+0000 7f25a9678900 20 run: stack=0x5636c4cee790 is io blocked 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d51c30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d51c30:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25a9678900 15 stack 0x5636c4cee790 end 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25a9678900 20 run: stack=0x5636c4cee790 is done 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d4a850:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25a9678900 15 stack 0x5636c4d3c450 end 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25a9678900 20 run: stack=0x5636c4d3c450 is done 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25a9678900 4 starting trim on bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d4e2c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d4e2c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25a9678900 20 enqueued request req=0x5636c4cd0430 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25a9678900 20 req: 0x5636c4cd0430 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25a9678900 20 run: stack=0x5636c4d3c450 is io blocked 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25637fe640 20 dequeued request req=0x5636c4cd0430 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25637fe640 20 RGWWQ: empty 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25637fe640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 : miss 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25637fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25637fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25637fe640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 info.flags=0x17 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25637fe640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 to cache LRU end 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25637fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25637fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25637fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25637fe640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-40 : miss 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25637fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25637fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25637fe640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-40 info.flags=0x0 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25637fe640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-40 to cache LRU end 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25637fe640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-40 : miss 2026-03-20T11:46:48.983 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.986+0000 7f25637fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4ce9a80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25637fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25637fe640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-40 info.flags=0x0 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25637fe640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-40 to cache LRU end 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25637fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1): adding source pipe: {s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25637fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1): adding dest pipe: {s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25637fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25637fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25637fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4ce9a80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d4e2c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4ce9a80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d4e2c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4ce9a80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d4e2c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4d44500:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 15 stack 0x5636c4d3f5a0 end 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 run: stack=0x5636c4d3f5a0 is done 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d4e2c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 collect(): s=0x5636c4d32ac0 stack=0x5636c4d3f5a0 is complete 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) markers=[,,,,,00000000002.36.11,,,00000000011.25.6,00000000003.20.6,], shards=11 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4cd9cb0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 10 trimming bilog shard 5 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000002.36.11 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 10 trimming bilog shard 8 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000011.25.6 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 10 trimming bilog shard 9 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000003.20.6 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4d44500:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.984 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 run: stack=0x5636c4d3c450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4ceb000:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d4eac0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 cr:s=0x5636c4873f10:op=0x5636c4cd8200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4ce9a80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4ceb000:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.5 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 run: stack=0x5636c4cee790 is io blocked 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d4eac0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.8 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 run: stack=0x5636c4884340 is io blocked 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 cr:s=0x5636c4873f10:op=0x5636c4cd8200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.9 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 run: stack=0x5636c4873f10 is io blocked 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4ce9a80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = ad96becb6d80d65de6805efa9ec7924392c02587a74c43b6f72ffd74331d3716 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:ad96becb6d80d65de6805efa9ec7924392c02587a74c43b6f72ffd74331d3716 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = e1a181c1789b34560a38cd850a7a9bb4b605b176403b7a215730f13b53ae70e6 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=e1a181c1789b34560a38cd850a7a9bb4b605b176403b7a215730f13b53ae70e6 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=38, curl_handle=0x5636c4cd9c00 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f2506ffd640 20 link_request req_data=0x5636c4ce1d00 req_data->id=38, curl_handle=0x5636c4cd9c00 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.987+0000 7f25a9678900 20 run: stack=0x5636c4d3f5a0 is io blocked 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 cr:s=0x5636c4873f10:op=0x5636c4cd8200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 cr:s=0x5636c4873f10:op=0x5636c4cd8200:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 15 stack 0x5636c4873f10 end 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 run: stack=0x5636c4873f10 is done 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4cd9cb0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4ceb000:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4ceb000:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 15 stack 0x5636c4cee790 end 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 run: stack=0x5636c4cee790 is done 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4cd9cb0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d4eac0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d4eac0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 15 stack 0x5636c4884340 end 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 run: stack=0x5636c4884340 is done 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4cd9cb0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 15 stack 0x5636c4d32ac0 end 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is done 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 4 starting trim on bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d51c30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d51c30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 enqueued request req=0x5636c4882640 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 req: 0x5636c4882640 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is io blocked 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f2562ffd640 20 dequeued request req=0x5636c4882640 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f2562ffd640 20 RGWWQ: empty 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f2562ffd640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 : miss 2026-03-20T11:46:48.986 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.988+0000 7f2562ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f2562ffd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f2562ffd640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 info.flags=0x17 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f2562ffd640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 to cache LRU end 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f2562ffd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f2562ffd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f2562ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f2562ffd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-18 : miss 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f2562ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f2562ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f2562ffd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-18 info.flags=0x0 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f2562ffd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-18 to cache LRU end 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f2562ffd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-18 : miss 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f2562ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f2562ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f2562ffd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-18 info.flags=0x0 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f2562ffd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-18 to cache LRU end 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f2562ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23): adding source pipe: {s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f2562ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23): adding dest pipe: {s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f2562ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f2562ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f2562ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d51c30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d51c30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d51c30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d51c30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d47930:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d507d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d507d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.987 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:48 +0000 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114648Z 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = c7cb3ac5c7fe764fe8c21aec80fe4ffb2698b1892664dc9ce9ce46cb7f3af2fd 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114648Z 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr:c7cb3ac5c7fe764fe8c21aec80fe4ffb2698b1892664dc9ce9ce46cb7f3af2fd 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = eeba640ce406a05ab7e89b9ef71815924978508ca59409e665381a91d872d904 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=eeba640ce406a05ab7e89b9ef71815924978508ca59409e665381a91d872d904 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114648Z 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=39, curl_handle=0x5636c4cdd120 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f2506ffd640 20 link_request req_data=0x5636c4cda310 req_data->id=39, curl_handle=0x5636c4cdd120 2026-03-20T11:46:48.988 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:48.989+0000 7f25a9678900 20 run: stack=0x5636c4884340 is io blocked 2026-03-20T11:46:49.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4ce5ff0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4ce5ff0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4ce5ff0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4ce5ff0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4d546f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 15 stack 0x5636c4886580 end 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 run: stack=0x5636c4886580 is done 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 collect(): s=0x5636c4cce390 stack=0x5636c4886580 is complete 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) markers=[00000000014.53.6,00000000008.38.6,00000000008.39.6,00000000008.35.6,00000000012.28.6,00000000010.38.6,00000000008.22.6,00000000010.27.6,00000000004.28.6,00000000008.20.6,00000000008.23.6], shards=11 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d546f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 10 trimming bilog shard 0 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000014.53.6 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 10 trimming bilog shard 1 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.38.6 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 10 trimming bilog shard 2 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.39.6 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 10 trimming bilog shard 3 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.35.6 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 10 trimming bilog shard 4 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000012.28.6 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 10 trimming bilog shard 5 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000010.38.6 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 10 trimming bilog shard 6 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.22.6 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 10 trimming bilog shard 7 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000010.27.6 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 10 trimming bilog shard 8 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000004.28.6 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 10 trimming bilog shard 9 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.20.6 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 10 trimming bilog shard 10 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.23.6 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4ceb000:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d3acb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c4873f10:op=0x5636c4ce5ff0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d4c630:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d57890:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4d56640:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4ccd190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c4d49930:op=0x5636c4d58d70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c4d54f40:op=0x5636c4d5a6f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c4876430:op=0x5636c4d5aef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c4d57190:op=0x5636c4d5b6f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4ceb000:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.0 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 run: stack=0x5636c4886580 is io blocked 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d3acb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.1 2026-03-20T11:46:49.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 run: stack=0x5636c4cee790 is io blocked 2026-03-20T11:46:49.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c4873f10:op=0x5636c4ce5ff0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.2 2026-03-20T11:46:49.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 run: stack=0x5636c4873f10 is io blocked 2026-03-20T11:46:49.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d4c630:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.3 2026-03-20T11:46:49.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 run: stack=0x5636c4ce56a0 is io blocked 2026-03-20T11:46:49.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d57890:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.4 2026-03-20T11:46:49.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is io blocked 2026-03-20T11:46:49.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4d56640:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.5 2026-03-20T11:46:49.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 run: stack=0x5636c48737c0 is io blocked 2026-03-20T11:46:49.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4ccd190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.6 2026-03-20T11:46:49.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 run: stack=0x5636c4ccdf90 is io blocked 2026-03-20T11:46:49.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c4d49930:op=0x5636c4d58d70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.7 2026-03-20T11:46:49.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 run: stack=0x5636c4d49930 is io blocked 2026-03-20T11:46:49.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c4d54f40:op=0x5636c4d5a6f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.8 2026-03-20T11:46:49.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 run: stack=0x5636c4d54f40 is io blocked 2026-03-20T11:46:49.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c4876430:op=0x5636c4d5aef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.9 2026-03-20T11:46:49.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 run: stack=0x5636c4876430 is io blocked 2026-03-20T11:46:49.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 cr:s=0x5636c4d57190:op=0x5636c4d5b6f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.10 2026-03-20T11:46:49.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.016+0000 7f25a9678900 20 run: stack=0x5636c4d57190 is io blocked 2026-03-20T11:46:49.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d57890:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d57890:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 15 stack 0x5636c4d3ab70 end 2026-03-20T11:46:49.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is done 2026-03-20T11:46:49.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d546f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4d56640:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4d56640:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 15 stack 0x5636c48737c0 end 2026-03-20T11:46:49.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 run: stack=0x5636c48737c0 is done 2026-03-20T11:46:49.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d546f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c4d49930:op=0x5636c4d58d70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c4d49930:op=0x5636c4d58d70:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 15 stack 0x5636c4d49930 end 2026-03-20T11:46:49.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 run: stack=0x5636c4d49930 is done 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d546f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c4d54f40:op=0x5636c4d5a6f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c4d54f40:op=0x5636c4d5a6f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 15 stack 0x5636c4d54f40 end 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 run: stack=0x5636c4d54f40 is done 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d546f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d4c630:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d4c630:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 15 stack 0x5636c4ce56a0 end 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 run: stack=0x5636c4ce56a0 is done 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d546f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d3acb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d3acb0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 15 stack 0x5636c4cee790 end 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 run: stack=0x5636c4cee790 is done 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c4d57190:op=0x5636c4d5b6f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c4d57190:op=0x5636c4d5b6f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 15 stack 0x5636c4d57190 end 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 run: stack=0x5636c4d57190 is done 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4ccd190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4ccd190:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 15 stack 0x5636c4ccdf90 end 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 run: stack=0x5636c4ccdf90 is done 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4ceb000:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4ceb000:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 15 stack 0x5636c4886580 end 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 run: stack=0x5636c4886580 is done 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d546f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c4876430:op=0x5636c4d5aef0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c4876430:op=0x5636c4d5aef0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 15 stack 0x5636c4876430 end 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.017 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 run: stack=0x5636c4876430 is done 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d546f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d546f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d546f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d546f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.018+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f25a9678900 20 cr:s=0x5636c4873f10:op=0x5636c4ce5ff0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f25a9678900 20 cr:s=0x5636c4873f10:op=0x5636c4ce5ff0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f25a9678900 15 stack 0x5636c4873f10 end 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f25a9678900 20 run: stack=0x5636c4873f10 is done 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4d546f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f25a9678900 20 cr:s=0x5636c4cce390:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f25a9678900 15 stack 0x5636c4cce390 end 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f25a9678900 20 run: stack=0x5636c4cce390 is done 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f25a9678900 4 starting trim on bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4d537e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4d537e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f25a9678900 20 enqueued request req=0x5636c4ccfa40 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f25a9678900 20 req: 0x5636c4ccfa40 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f25a9678900 20 run: stack=0x5636c4ccdf90 is io blocked 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f2561ffb640 20 dequeued request req=0x5636c4ccfa40 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f2561ffb640 20 RGWWQ: empty 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f2561ffb640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 : miss 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f2561ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f2561ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f2561ffb640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 info.flags=0x17 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f2561ffb640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 to cache LRU end 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f2561ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f2561ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f2561ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f2561ffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-50 : miss 2026-03-20T11:46:49.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f2561ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f2561ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f2561ffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-50 info.flags=0x0 2026-03-20T11:46:49.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f2561ffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-50 to cache LRU end 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f2561ffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-50 : miss 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.019+0000 7f2561ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d50e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f2561ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f2561ffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-50 info.flags=0x0 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f2561ffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-50 to cache LRU end 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d50e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d50e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4d50e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4cef030:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 15 stack 0x5636c4873b00 end 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 run: stack=0x5636c4873b00 is done 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 collect(): s=0x5636c4cd5900 stack=0x5636c4873b00 is complete 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) markers=[00000000014.49.6,00000000008.48.6,00000000008.43.6,00000000008.23.6,00000000012.30.6,00000000010.25.6,00000000008.36.6,00000000010.29.6,00000000004.42.6,00000000008.28.6,00000000008.26.6], shards=11 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d56b00:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 10 trimming bilog shard 0 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000014.49.6 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 10 trimming bilog shard 1 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.48.6 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 10 trimming bilog shard 2 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.43.6 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 10 trimming bilog shard 3 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.23.6 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 10 trimming bilog shard 4 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000012.30.6 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 10 trimming bilog shard 5 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000010.25.6 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 10 trimming bilog shard 6 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.36.6 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 10 trimming bilog shard 7 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000010.29.6 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 10 trimming bilog shard 8 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000004.42.6 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 10 trimming bilog shard 9 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.28.6 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 10 trimming bilog shard 10 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.26.6 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4ceb000:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4ceba10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4874680:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4ce5ff0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c4d49930:op=0x5636c4d4e610:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4ccd190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d58190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c4d52280:op=0x5636c4d58990:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c4ce5af0:op=0x5636c4d59190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c4d57190:op=0x5636c4d5a550:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c4cdcee0:op=0x5636c4d5ad50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4ceb000:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.0 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 run: stack=0x5636c4873b00 is io blocked 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4ceba10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.1 2026-03-20T11:46:49.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f2561ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10): adding source pipe: {s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 run: stack=0x5636c4cee790 is io blocked 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4874680:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.2 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 run: stack=0x5636c4886580 is io blocked 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4ce5ff0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.3 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 run: stack=0x5636c4ce56a0 is io blocked 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c4d49930:op=0x5636c4d4e610:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.4 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 run: stack=0x5636c4d49930 is io blocked 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4ccd190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.5 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 run: stack=0x5636c48737c0 is io blocked 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d58190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.6 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is io blocked 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c4d52280:op=0x5636c4d58990:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.7 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f2561ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10): adding dest pipe: {s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 run: stack=0x5636c4d52280 is io blocked 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c4ce5af0:op=0x5636c4d59190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.8 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 run: stack=0x5636c4ce5af0 is io blocked 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c4d57190:op=0x5636c4d5a550:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.9 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 run: stack=0x5636c4d57190 is io blocked 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 cr:s=0x5636c4cdcee0:op=0x5636c4d5ad50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.10 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f25a9678900 20 run: stack=0x5636c4cdcee0 is io blocked 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f2561ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f2561ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.020+0000 7f2561ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4d537e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4d537e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4d537e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4d537e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 cr:s=0x5636c4d55c20:op=0x5636c4d550f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 run: stack=0x5636c4ccdf90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 cr:s=0x5636c4d55c20:op=0x5636c4d50e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 cr:s=0x5636c4d55c20:op=0x5636c4d50e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = ea034987a339e8ab7cd4d64236de51c5d5dc5dd8290f8a0903f2015b8f8d7401 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114649Z 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:ea034987a339e8ab7cd4d64236de51c5d5dc5dd8290f8a0903f2015b8f8d7401 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = a87812412aed42e8d7eeb0f03c7895889d0b37808b89cbc4447f2020a0118c7c 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a87812412aed42e8d7eeb0f03c7895889d0b37808b89cbc4447f2020a0118c7c 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114649Z 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=40, curl_handle=0x5636c4ce15f0 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 run: stack=0x5636c4d55c20 is io blocked 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 cr:s=0x5636c4d49930:op=0x5636c4d4e610:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 cr:s=0x5636c4d49930:op=0x5636c4d4e610:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 15 stack 0x5636c4d49930 end 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 run: stack=0x5636c4d49930 is done 2026-03-20T11:46:49.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d56b00:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 cr:s=0x5636c4d57190:op=0x5636c4d5a550:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 cr:s=0x5636c4d57190:op=0x5636c4d5a550:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 15 stack 0x5636c4d57190 end 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 run: stack=0x5636c4d57190 is done 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d56b00:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.021+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f2506ffd640 20 link_request req_data=0x5636c4886970 req_data->id=40, curl_handle=0x5636c4ce15f0 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4ce5af0:op=0x5636c4d59190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4ce5af0:op=0x5636c4d59190:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 15 stack 0x5636c4ce5af0 end 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 run: stack=0x5636c4ce5af0 is done 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d56b00:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4ceb000:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4ceb000:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 15 stack 0x5636c4873b00 end 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 run: stack=0x5636c4873b00 is done 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d58190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d58190:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 15 stack 0x5636c4d3ab70 end 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is done 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4cdcee0:op=0x5636c4d5ad50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4cdcee0:op=0x5636c4d5ad50:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 15 stack 0x5636c4cdcee0 end 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 run: stack=0x5636c4cdcee0 is done 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4874680:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4874680:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 15 stack 0x5636c4886580 end 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 run: stack=0x5636c4886580 is done 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4ccd190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4ccd190:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 15 stack 0x5636c48737c0 end 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 run: stack=0x5636c48737c0 is done 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4ce5ff0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4ce5ff0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 15 stack 0x5636c4ce56a0 end 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 run: stack=0x5636c4ce56a0 is done 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4d52280:op=0x5636c4d58990:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4d52280:op=0x5636c4d58990:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 15 stack 0x5636c4d52280 end 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 run: stack=0x5636c4d52280 is done 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4ceba10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4ceba10:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 15 stack 0x5636c4cee790 end 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 run: stack=0x5636c4cee790 is done 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d56b00:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d56b00:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d56b00:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d56b00:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d56b00:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d56b00:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d56b00:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4d56b00:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4cd5900:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 15 stack 0x5636c4cd5900 end 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 run: stack=0x5636c4cd5900 is done 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 4 starting trim on bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d537e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d537e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 enqueued request req=0x5636c4ce5e00 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 req: 0x5636c4ce5e00 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is io blocked 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25617fa640 20 dequeued request req=0x5636c4ce5e00 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25617fa640 20 RGWWQ: empty 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25617fa640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 : miss 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.022+0000 7f25617fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25617fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25617fa640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 info.flags=0x17 2026-03-20T11:46:49.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25617fa640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 to cache LRU end 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25617fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25617fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25617fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25617fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-24 : miss 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25617fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25617fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25617fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-24 info.flags=0x0 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25617fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-24 to cache LRU end 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25617fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-24 : miss 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25617fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25617fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25617fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-24 info.flags=0x0 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25617fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-24 to cache LRU end 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25617fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29): adding source pipe: {s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25617fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29): adding dest pipe: {s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25617fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25617fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25617fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d537e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d537e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d537e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d537e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4d56ad0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 7edb0451c52e2ad0c69831fbb7e6f3437157194fbb19721d3918b46bf02eac0b 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114649Z 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:7edb0451c52e2ad0c69831fbb7e6f3437157194fbb19721d3918b46bf02eac0b 2026-03-20T11:46:49.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:49.027 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:49.027 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:49.027 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:49.027 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = b199251ae1f7ab270b1c3297ae47080b0c4fb7177c83ef933b528c27333abf3e 2026-03-20T11:46:49.027 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b199251ae1f7ab270b1c3297ae47080b0c4fb7177c83ef933b528c27333abf3e 2026-03-20T11:46:49.027 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.027 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114649Z 2026-03-20T11:46:49.027 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:49.027 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=41, curl_handle=0x5636c4884b90 2026-03-20T11:46:49.027 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f25a9678900 20 run: stack=0x5636c48737c0 is io blocked 2026-03-20T11:46:49.027 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.023+0000 7f2506ffd640 20 link_request req_data=0x5636c4cdd870 req_data->id=41, curl_handle=0x5636c4884b90 2026-03-20T11:46:49.027 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:49.024+0000 7f129aa25640 -1 req 6815710435845787374 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:49.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4ce9a80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4ce9a80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d507d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4ce9a80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d507d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4ce9a80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d507d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4d44500:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 15 stack 0x5636c4d3f5a0 end 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 20 run: stack=0x5636c4d3f5a0 is done 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d507d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 20 collect(): s=0x5636c4d3c450 stack=0x5636c4d3f5a0 is complete 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-40[4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1]) markers=[,,,,,00000000001.144.6,,,,,], shards=11 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d47930:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 15 stack 0x5636c4884340 end 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 20 run: stack=0x5636c4884340 is done 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d4a850:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 10 trimming bilog shard 5 of :dipepu-40[4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1]) at marker 00000000001.144.6 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 20 run: stack=0x5636c4d3c450 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 20 collect(): s=0x5636c4d32ac0 stack=0x5636c4884340 is complete 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-18[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23]) markers=[,,,,,,,,00000000002.59.6,,], shards=11 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4ce5ff0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d547d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 10 trimming bilog shard 8 of :dipepu-18[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23]) at marker 00000000002.59.6 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4ce5ff0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1.5 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 20 run: stack=0x5636c4d3f5a0 is io blocked 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cd8200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cd8200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23.8 2026-03-20T11:46:49.029 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.032+0000 7f25a9678900 20 run: stack=0x5636c4884340 is io blocked 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cd8200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cd8200:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 15 stack 0x5636c4884340 end 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 run: stack=0x5636c4884340 is done 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d547d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 15 stack 0x5636c4d32ac0 end 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is done 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 4 starting trim on bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d489a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d489a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 enqueued request req=0x5636c4ce5900 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 req: 0x5636c4ce5900 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f2560ff9640 20 dequeued request req=0x5636c4ce5900 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f2560ff9640 20 RGWWQ: empty 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f2560ff9640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 : miss 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f2560ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is io blocked 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4ce5ff0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4ce5ff0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 15 stack 0x5636c4d3f5a0 end 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 run: stack=0x5636c4d3f5a0 is done 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4d4a850:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 cr:s=0x5636c4d3c450:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 15 stack 0x5636c4d3c450 end 2026-03-20T11:46:49.030 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 run: stack=0x5636c4d3c450 is done 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 4 starting trim on bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4d537e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4d537e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 enqueued request req=0x5636c4cd0430 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 req: 0x5636c4cd0430 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25a9678900 20 run: stack=0x5636c4d3f5a0 is io blocked 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25607f8640 20 dequeued request req=0x5636c4cd0430 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25607f8640 20 RGWWQ: empty 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25607f8640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 : miss 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.033+0000 7f25607f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f2560ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f2560ff9640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 info.flags=0x17 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f2560ff9640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 to cache LRU end 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f2560ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f2560ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f2560ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f2560ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-56 : miss 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f2560ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25607f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25607f8640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 info.flags=0x17 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25607f8640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 to cache LRU end 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25607f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25607f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25607f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25607f8640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-27 : miss 2026-03-20T11:46:49.031 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25607f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f2560ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f2560ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-56 info.flags=0x0 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f2560ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-56 to cache LRU end 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f2560ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-56 : miss 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f2560ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25607f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25607f8640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-27 info.flags=0x0 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25607f8640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-27 to cache LRU end 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25607f8640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-27 : miss 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25607f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f2560ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f2560ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-56 info.flags=0x0 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f2560ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-56 to cache LRU end 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f2560ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16): adding source pipe: {s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f2560ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16): adding dest pipe: {s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f2560ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f2560ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f2560ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d489a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d489a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d489a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d489a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4ce9a80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cef030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cef030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25607f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 59700ae2706e38d8d0b67125cc8a772892915fc46cd4f40e2dad9a7f1e8f4c3d 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25607f8640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-27 info.flags=0x0 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25607f8640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-27 to cache LRU end 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114649Z 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:59700ae2706e38d8d0b67125cc8a772892915fc46cd4f40e2dad9a7f1e8f4c3d 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25607f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32): adding source pipe: {s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25607f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32): adding dest pipe: {s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:49.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 004f8e256c918b43d024d697858a66e06ca63a06ed70a1723d7cde4110cfba88 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=004f8e256c918b43d024d697858a66e06ca63a06ed70a1723d7cde4110cfba88 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114649Z 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=42, curl_handle=0x5636c4cdd120 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25607f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 run: stack=0x5636c4884340 is io blocked 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25607f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25607f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4d537e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4d537e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4d537e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4d537e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f2506ffd640 20 link_request req_data=0x5636c4cda310 req_data->id=42, curl_handle=0x5636c4cdd120 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d44500:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 run: stack=0x5636c4d3f5a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d47930:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d47930:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 90b8d3b8c083bf52081485c22c4ce9922d50bcde60a5ff537c7749c2b7516707 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114649Z 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:90b8d3b8c083bf52081485c22c4ce9922d50bcde60a5ff537c7749c2b7516707 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = b770cce8fbd2902a45aa4c5587d538679d285c93445ab377f411aa7b2d145f9f 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b770cce8fbd2902a45aa4c5587d538679d285c93445ab377f411aa7b2d145f9f 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114649Z 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=43, curl_handle=0x5636c4cd9c00 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f25a9678900 20 run: stack=0x5636c4ce56a0 is io blocked 2026-03-20T11:46:49.033 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.034+0000 7f2506ffd640 20 link_request req_data=0x5636c4ce1d00 req_data->id=43, curl_handle=0x5636c4cd9c00 2026-03-20T11:46:49.034 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:49.034+0000 7f12769dd640 -1 req 15067869551570722230 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:49.034 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:49.035+0000 7f12771de640 -1 req 8977227229294815884 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:49.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f25a9678900 20 cr:s=0x5636c4d55c20:op=0x5636c4d50e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f25a9678900 20 cr:s=0x5636c4d55c20:op=0x5636c4d50e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f25a9678900 20 cr:s=0x5636c4d55c20:op=0x5636c4d50e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f25a9678900 20 cr:s=0x5636c4d55c20:op=0x5636c4d50e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f25a9678900 20 cr:s=0x5636c4d55c20:op=0x5636c4d550f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f25a9678900 15 stack 0x5636c4d55c20 end 2026-03-20T11:46:49.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f25a9678900 20 run: stack=0x5636c4d55c20 is done 2026-03-20T11:46:49.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f25a9678900 20 collect(): s=0x5636c4ccdf90 stack=0x5636c4d55c20 is complete 2026-03-20T11:46:49.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:49.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f25a9678900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:49.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f25a9678900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:49.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:49.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f25a9678900 15 stack 0x5636c4ccdf90 end 2026-03-20T11:46:49.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f25a9678900 20 stack->operate() returned ret=-22 2026-03-20T11:46:49.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f25a9678900 20 run: stack=0x5636c4ccdf90 is done 2026-03-20T11:46:49.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:49.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f25a9678900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:49.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f25a9678900 4 starting trim on bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:46:49.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4d537e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4d537e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f25a9678900 20 enqueued request req=0x5636c4ccfa40 2026-03-20T11:46:49.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:49.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f25a9678900 20 req: 0x5636c4ccfa40 2026-03-20T11:46:49.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f255fff7640 20 dequeued request req=0x5636c4ccfa40 2026-03-20T11:46:49.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f255fff7640 20 RGWWQ: empty 2026-03-20T11:46:49.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f255fff7640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 : miss 2026-03-20T11:46:49.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f255fff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f25a9678900 20 run: stack=0x5636c4ccdf90 is io blocked 2026-03-20T11:46:49.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f255fff7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:49.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f255fff7640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 info.flags=0x17 2026-03-20T11:46:49.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f255fff7640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 to cache LRU end 2026-03-20T11:46:49.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f255fff7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:49.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f255fff7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:49.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f255fff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:46:49.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f255fff7640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-44 : miss 2026-03-20T11:46:49.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f255fff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f255fff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f255fff7640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-44 info.flags=0x0 2026-03-20T11:46:49.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f255fff7640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-44 to cache LRU end 2026-03-20T11:46:49.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f255fff7640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-44 : miss 2026-03-20T11:46:49.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f255fff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f255fff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f255fff7640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-44 info.flags=0x0 2026-03-20T11:46:49.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f255fff7640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-44 to cache LRU end 2026-03-20T11:46:49.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f255fff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4): adding source pipe: {s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.063+0000 7f255fff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4): adding dest pipe: {s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f255fff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f255fff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f255fff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:46:49.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4d537e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4d537e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4d537e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4d537e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 20 cr:s=0x5636c4d55c20:op=0x5636c4d550f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 20 run: stack=0x5636c4ccdf90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 20 cr:s=0x5636c4d55c20:op=0x5636c4d537e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 20 cr:s=0x5636c4d55c20:op=0x5636c4d537e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:49.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:49.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.062 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 6b9d08cdc4d3e03e956d57e2b4e92428cf3dc2cb12653d90ba63cdb300a37c4c 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114649Z 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:6b9d08cdc4d3e03e956d57e2b4e92428cf3dc2cb12653d90ba63cdb300a37c4c 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = e53bbb3637fd44555af59ce67b8a2c45a10c711366162e6001dc195fb4c0e5d5 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=e53bbb3637fd44555af59ce67b8a2c45a10c711366162e6001dc195fb4c0e5d5 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114649Z 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=44, curl_handle=0x5636c4ce15f0 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f25a9678900 20 run: stack=0x5636c4d55c20 is io blocked 2026-03-20T11:46:49.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.064+0000 7f2506ffd640 20 link_request req_data=0x5636c4886970 req_data->id=44, curl_handle=0x5636c4ce15f0 2026-03-20T11:46:49.063 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:49.064+0000 7f12791e2640 -1 req 12089446633053588258 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:49.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:49.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4d56ad0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f25a9678900 15 stack 0x5636c48737c0 end 2026-03-20T11:46:49.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f25a9678900 20 run: stack=0x5636c48737c0 is done 2026-03-20T11:46:49.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f25a9678900 20 collect(): s=0x5636c4d3ab70 stack=0x5636c48737c0 is complete 2026-03-20T11:46:49.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:49.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f25a9678900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:49.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f25a9678900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:49.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:49.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f25a9678900 15 stack 0x5636c4d3ab70 end 2026-03-20T11:46:49.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f25a9678900 20 stack->operate() returned ret=-22 2026-03-20T11:46:49.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is done 2026-03-20T11:46:49.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f25a9678900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f25a9678900 4 starting trim on bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d547d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d547d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f25a9678900 20 enqueued request req=0x5636c4ce5e00 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f25a9678900 20 req: 0x5636c4ce5e00 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is io blocked 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f255f7f6640 20 dequeued request req=0x5636c4ce5e00 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f255f7f6640 20 RGWWQ: empty 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f255f7f6640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 : miss 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f255f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f255f7f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f255f7f6640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 info.flags=0x17 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f255f7f6640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 to cache LRU end 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f255f7f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f255f7f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f255f7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f255f7f6640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-17 : miss 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f255f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f255f7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f255f7f6640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-17 info.flags=0x0 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f255f7f6640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-17 to cache LRU end 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f255f7f6640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-17 : miss 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.065+0000 7f255f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f255f7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f255f7f6640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-17 info.flags=0x0 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f255f7f6640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-17 to cache LRU end 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f255f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22): adding source pipe: {s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f255f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22): adding dest pipe: {s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f255f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f255f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f255f7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d547d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d547d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d547d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d547d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4d56ad0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:49.065 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:49.066 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:49.066 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.066 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:49.066 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.066 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.066 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.066 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:49.066 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = d39d71ed860891694d0adf8a7cf5165aa54a669c2e160a92d8dab81417f242d3 2026-03-20T11:46:49.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:49.066 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114649Z 2026-03-20T11:46:49.066 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:49.066 INFO:teuthology.orchestra.run.vm02.stderr:d39d71ed860891694d0adf8a7cf5165aa54a669c2e160a92d8dab81417f242d3 2026-03-20T11:46:49.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:49.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:49.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:49.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:49.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 401a849176df1851fca68c936b59835271f936ca4b9dc64dc930d0d11782d8d9 2026-03-20T11:46:49.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=401a849176df1851fca68c936b59835271f936ca4b9dc64dc930d0d11782d8d9 2026-03-20T11:46:49.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114649Z 2026-03-20T11:46:49.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:49.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=45, curl_handle=0x5636c4884b90 2026-03-20T11:46:49.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f25a9678900 20 run: stack=0x5636c48737c0 is io blocked 2026-03-20T11:46:49.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.066+0000 7f2506ffd640 20 link_request req_data=0x5636c4cdd870 req_data->id=45, curl_handle=0x5636c4884b90 2026-03-20T11:46:49.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cef030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cef030:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:49.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4ce9a80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 15 stack 0x5636c4884340 end 2026-03-20T11:46:49.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 run: stack=0x5636c4884340 is done 2026-03-20T11:46:49.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d47930:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d47930:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:49.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.073 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 collect(): s=0x5636c4d32ac0 stack=0x5636c4884340 is complete 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 15 stack 0x5636c4d32ac0 end 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 stack->operate() returned ret=-22 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is done 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d44500:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 15 stack 0x5636c4ce56a0 end 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 run: stack=0x5636c4ce56a0 is done 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 collect(): s=0x5636c4d3f5a0 stack=0x5636c4ce56a0 is complete 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 15 stack 0x5636c4d3f5a0 end 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 stack->operate() returned ret=-22 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 run: stack=0x5636c4d3f5a0 is done 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 4 starting trim on bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d489a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 4 starting trim on bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d489a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 enqueued request req=0x5636c4cd64a0 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 req: 0x5636c4cd64a0 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is io blocked 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4ce37b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4ce37b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 enqueued request req=0x5636c4cd9410 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 req: 0x5636c4cd64a0 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 req: 0x5636c4cd9410 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f255e7f4640 20 dequeued request req=0x5636c4cd64a0 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f255e7f4640 20 RGWWQ: 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f255e7f4640 20 req: 0x5636c4cd9410 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f255e7f4640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 : miss 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f255e7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f255eff5640 20 dequeued request req=0x5636c4cd9410 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f255eff5640 20 RGWWQ: empty 2026-03-20T11:46:49.074 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f255eff5640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 : miss 2026-03-20T11:46:49.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f255eff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f25a9678900 20 run: stack=0x5636c4d3f5a0 is io blocked 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f255e7f4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f255e7f4640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 info.flags=0x17 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f255e7f4640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 to cache LRU end 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f255e7f4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f255e7f4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.076+0000 7f255e7f4640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255e7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255e7f4640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-46 : miss 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255e7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255eff5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255eff5640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 info.flags=0x17 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255eff5640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 to cache LRU end 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255eff5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255eff5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255eff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255eff5640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-37 : miss 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255eff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255e7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255e7f4640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-46 info.flags=0x0 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255e7f4640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-46 to cache LRU end 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255e7f4640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-46 : miss 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255e7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255eff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255eff5640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-37 info.flags=0x0 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255eff5640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-37 to cache LRU end 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255eff5640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-37 : miss 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255eff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255e7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255e7f4640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-46 info.flags=0x0 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255e7f4640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-46 to cache LRU end 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255e7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6): adding source pipe: {s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255e7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6): adding dest pipe: {s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255e7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255e7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255e7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d489a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d489a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d489a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d489a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d44500:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.076 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4ce9a80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4ce9a80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 8e4169e144e67a11d9a681e27e6d8bac31fcfe20410875f9300545a103c73d1c 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114649Z 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:8e4169e144e67a11d9a681e27e6d8bac31fcfe20410875f9300545a103c73d1c 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 94055ff1e5366fcd2982b9df31335abf4c3b7b2bb62ebdd4bb2b1a0ae57bdfe4 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=94055ff1e5366fcd2982b9df31335abf4c3b7b2bb62ebdd4bb2b1a0ae57bdfe4 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114649Z 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255eff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255eff5640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-37 info.flags=0x0 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255eff5640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-37 to cache LRU end 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255eff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42): adding source pipe: {s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255eff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42): adding dest pipe: {s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255eff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255eff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f255eff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=46, curl_handle=0x5636c4cd9c00 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 run: stack=0x5636c4ce56a0 is io blocked 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4ce37b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4ce37b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4ce37b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4ce37b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cef030:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f2506ffd640 20 link_request req_data=0x5636c4ce1d00 req_data->id=46, curl_handle=0x5636c4cd9c00 2026-03-20T11:46:49.077 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 run: stack=0x5636c4d3f5a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d47930:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d47930:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = c38eddceaa1d0057d7eb688b3b76c01c9d6760af658a2d42b6218d1aa99214cd 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114649Z 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:c38eddceaa1d0057d7eb688b3b76c01c9d6760af658a2d42b6218d1aa99214cd 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 51154baae6a9509a13ae5d53917121f212bf8a70e17aebca608c3b8b3c4b5ae8 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=51154baae6a9509a13ae5d53917121f212bf8a70e17aebca608c3b8b3c4b5ae8 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114649Z 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=47, curl_handle=0x5636c4cdd120 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f25a9678900 20 run: stack=0x5636c4884340 is io blocked 2026-03-20T11:46:49.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.077+0000 7f2506ffd640 20 link_request req_data=0x5636c4cda310 req_data->id=47, curl_handle=0x5636c4cdd120 2026-03-20T11:46:49.078 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:49.078+0000 7f12789e1640 -1 req 15467902278786793919 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f25a9678900 20 cr:s=0x5636c4d55c20:op=0x5636c4d537e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f25a9678900 20 cr:s=0x5636c4d55c20:op=0x5636c4d537e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f25a9678900 20 cr:s=0x5636c4d55c20:op=0x5636c4d550f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f25a9678900 15 stack 0x5636c4d55c20 end 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f25a9678900 20 run: stack=0x5636c4d55c20 is done 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f25a9678900 20 collect(): s=0x5636c4ccdf90 stack=0x5636c4d55c20 is complete 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f25a9678900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f25a9678900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f25a9678900 15 stack 0x5636c4ccdf90 end 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f25a9678900 20 stack->operate() returned ret=-22 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f25a9678900 20 run: stack=0x5636c4ccdf90 is done 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f25a9678900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f25a9678900 4 starting trim on bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4d489a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4d489a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f25a9678900 20 enqueued request req=0x5636c4ccfa40 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f25a9678900 20 req: 0x5636c4ccfa40 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f255dff3640 20 dequeued request req=0x5636c4ccfa40 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f255dff3640 20 RGWWQ: empty 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f255dff3640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 : miss 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f255dff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f25a9678900 20 run: stack=0x5636c4ccdf90 is io blocked 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f255dff3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f255dff3640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 info.flags=0x17 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f255dff3640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 to cache LRU end 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f255dff3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f255dff3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f255dff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f255dff3640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-1 : miss 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.105+0000 7f255dff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f255dff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f255dff3640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-1 info.flags=0x0 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f255dff3640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-1 to cache LRU end 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f255dff3640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-1 : miss 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f255dff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f255dff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f255dff3640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-1 info.flags=0x0 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f255dff3640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-1 to cache LRU end 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f255dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1): adding source pipe: {s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f255dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1): adding dest pipe: {s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f255dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f255dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f255dff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4d489a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4d489a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4d489a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4d489a0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 20 cr:s=0x5636c4d55c20:op=0x5636c4d537e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 20 run: stack=0x5636c4ccdf90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 20 cr:s=0x5636c4d55c20:op=0x5636c4d55090:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 20 cr:s=0x5636c4d55c20:op=0x5636c4d55090:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:49.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = fc2fda705165fcfdbeb2f030c5d7efd715871f28d6672a5bc97a606abe3d8a5c 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114649Z 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:fc2fda705165fcfdbeb2f030c5d7efd715871f28d6672a5bc97a606abe3d8a5c 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 466cfa06d0a115b9ef6cb94017295553bc8fd1256c01d2b1b2fac0bb0befa299 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=466cfa06d0a115b9ef6cb94017295553bc8fd1256c01d2b1b2fac0bb0befa299 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114649Z 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=48, curl_handle=0x5636c4ce15f0 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f25a9678900 20 run: stack=0x5636c4d55c20 is io blocked 2026-03-20T11:46:49.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.106+0000 7f2506ffd640 20 link_request req_data=0x5636c4886970 req_data->id=48, curl_handle=0x5636c4ce15f0 2026-03-20T11:46:49.104 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:49.107+0000 7f12609b1640 -1 req 2769838963337554422 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:49.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.109+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.109+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.109+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.109+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.109+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4d56ad0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.109+0000 7f25a9678900 15 stack 0x5636c48737c0 end 2026-03-20T11:46:49.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.109+0000 7f25a9678900 20 run: stack=0x5636c48737c0 is done 2026-03-20T11:46:49.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.109+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.109+0000 7f25a9678900 20 collect(): s=0x5636c4d3ab70 stack=0x5636c48737c0 is complete 2026-03-20T11:46:49.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.109+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:49.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.109+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-17[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22]) markers=[,,,,,,,,00000000005.51.18,,], shards=11 2026-03-20T11:46:49.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.109+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d56ad0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.109+0000 7f25a9678900 10 trimming bilog shard 8 of :dipepu-17[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22]) at marker 00000000005.51.18 2026-03-20T11:46:49.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.109+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.109+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4ceba10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.109+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4ceba10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.109+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22.8 2026-03-20T11:46:49.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.109+0000 7f25a9678900 20 run: stack=0x5636c48737c0 is io blocked 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4ceba10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4ceba10:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f25a9678900 15 stack 0x5636c48737c0 end 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f25a9678900 20 run: stack=0x5636c48737c0 is done 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d56ad0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f25a9678900 15 stack 0x5636c4d3ab70 end 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is done 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f25a9678900 4 starting trim on bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4ceba10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4ceba10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f25a9678900 20 enqueued request req=0x5636c4ce5e00 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f25a9678900 20 RGWWQ: 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f25a9678900 20 req: 0x5636c4ce5e00 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f255d7f2640 20 dequeued request req=0x5636c4ce5e00 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f255d7f2640 20 RGWWQ: empty 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f255d7f2640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 : miss 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f255d7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is io blocked 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f255d7f2640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f255d7f2640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 info.flags=0x17 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f255d7f2640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 to cache LRU end 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f255d7f2640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f255d7f2640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f255d7f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:46:49.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f255d7f2640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-7 : miss 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.110+0000 7f255d7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f255d7f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f255d7f2640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-7 info.flags=0x0 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f255d7f2640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-7 to cache LRU end 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f255d7f2640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-7 : miss 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f255d7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f255d7f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f255d7f2640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-7 info.flags=0x0 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f255d7f2640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-7 to cache LRU end 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f255d7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12): adding source pipe: {s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f255d7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12): adding dest pipe: {s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f255d7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f255d7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f255d7f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4ceba10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4ceba10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4ceba10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4ceba10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4d56ad0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:49.108 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:49.109 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.109 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:49.109 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.109 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.109 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.109 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:49.109 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = ca27b54771c26fb83cffeb74454e006ba0f3eeaf2d083f67265d0aaf4210de74 2026-03-20T11:46:49.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:49.109 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114649Z 2026-03-20T11:46:49.109 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:49.109 INFO:teuthology.orchestra.run.vm02.stderr:ca27b54771c26fb83cffeb74454e006ba0f3eeaf2d083f67265d0aaf4210de74 2026-03-20T11:46:49.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:49.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:49.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:49.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:49.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = b385a0ff2607fde6e61c7f5a722a75da93ecc48c58291ad14a7078cbdc0a6405 2026-03-20T11:46:49.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b385a0ff2607fde6e61c7f5a722a75da93ecc48c58291ad14a7078cbdc0a6405 2026-03-20T11:46:49.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114649Z 2026-03-20T11:46:49.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:49.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 20 register_request mgr=0x7ffeded66720 req_data->id=49, curl_handle=0x5636c4884b90 2026-03-20T11:46:49.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f25a9678900 20 run: stack=0x5636c48737c0 is io blocked 2026-03-20T11:46:49.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.111+0000 7f2506ffd640 20 link_request req_data=0x5636c4cdd870 req_data->id=49, curl_handle=0x5636c4884b90 2026-03-20T11:46:49.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.119+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4ce9a80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.119+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4ce9a80:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:49.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.119+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d44500:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.119+0000 7f25a9678900 15 stack 0x5636c4ce56a0 end 2026-03-20T11:46:49.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.119+0000 7f25a9678900 20 run: stack=0x5636c4ce56a0 is done 2026-03-20T11:46:49.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.119+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.119+0000 7f25a9678900 20 collect(): s=0x5636c4d32ac0 stack=0x5636c4ce56a0 is complete 2026-03-20T11:46:49.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.119+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:49.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.119+0000 7f25a9678900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:49.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.119+0000 7f25a9678900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:49.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.119+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d45040:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:49.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.119+0000 7f25a9678900 15 stack 0x5636c4d32ac0 end 2026-03-20T11:46:49.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.119+0000 7f25a9678900 20 stack->operate() returned ret=-22 2026-03-20T11:46:49.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.119+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is done 2026-03-20T11:46:49.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.119+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:49.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.119+0000 7f25a9678900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:49.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.119+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.120+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d47930:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.120+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d47930:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.120+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d47930:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.120+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d47930:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.120+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4cef030:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.120+0000 7f25a9678900 15 stack 0x5636c4884340 end 2026-03-20T11:46:49.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.120+0000 7f25a9678900 20 run: stack=0x5636c4884340 is done 2026-03-20T11:46:49.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.120+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.120+0000 7f25a9678900 20 collect(): s=0x5636c4d3f5a0 stack=0x5636c4884340 is complete 2026-03-20T11:46:49.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.120+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:49.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.120+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-37[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42]) markers=[,,,,,,,,,,00000000001.45.6], shards=11 2026-03-20T11:46:49.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.120+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4d4fbc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.120+0000 7f25a9678900 10 trimming bilog shard 10 of :dipepu-37[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42]) at marker 00000000001.45.6 2026-03-20T11:46:49.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.120+0000 7f25a9678900 20 run: stack=0x5636c4d3f5a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.120+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d4ca20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.120+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d4ca20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.120+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42.10 2026-03-20T11:46:49.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.120+0000 7f25a9678900 20 run: stack=0x5636c4884340 is io blocked 2026-03-20T11:46:49.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.121+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d4ca20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.121+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4d4ca20:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.121+0000 7f25a9678900 15 stack 0x5636c4884340 end 2026-03-20T11:46:49.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.121+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.121+0000 7f25a9678900 20 run: stack=0x5636c4884340 is done 2026-03-20T11:46:49.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.121+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4d4fbc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.121+0000 7f25a9678900 20 cr:s=0x5636c4d3f5a0:op=0x5636c4cd2010:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.121+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:46:49.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.121+0000 7f25a9678900 15 stack 0x5636c4d3f5a0 end 2026-03-20T11:46:49.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.121+0000 7f25a9678900 20 run: stack=0x5636c4d3f5a0 is done 2026-03-20T11:46:49.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.121+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:49.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.121+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.148+0000 7f25a9678900 20 cr:s=0x5636c4d55c20:op=0x5636c4d55090:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.148+0000 7f25a9678900 20 cr:s=0x5636c4d55c20:op=0x5636c4d55090:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:49.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.148+0000 7f25a9678900 20 cr:s=0x5636c4d55c20:op=0x5636c4d537e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.148+0000 7f25a9678900 15 stack 0x5636c4d55c20 end 2026-03-20T11:46:49.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.148+0000 7f25a9678900 20 run: stack=0x5636c4d55c20 is done 2026-03-20T11:46:49.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.148+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.148+0000 7f25a9678900 20 collect(): s=0x5636c4ccdf90 stack=0x5636c4d55c20 is complete 2026-03-20T11:46:49.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.148+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:49.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.148+0000 7f25a9678900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:49.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.148+0000 7f25a9678900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:49.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.148+0000 7f25a9678900 20 cr:s=0x5636c4ccdf90:op=0x5636c4cd32e0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:49.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.148+0000 7f25a9678900 15 stack 0x5636c4ccdf90 end 2026-03-20T11:46:49.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.148+0000 7f25a9678900 20 stack->operate() returned ret=-22 2026-03-20T11:46:49.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.148+0000 7f25a9678900 20 run: stack=0x5636c4ccdf90 is done 2026-03-20T11:46:49.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.148+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:49.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.148+0000 7f25a9678900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:49.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.148+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4cdd140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 cr:s=0x5636c48737c0:op=0x5636c4d56ad0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 15 stack 0x5636c48737c0 end 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 run: stack=0x5636c48737c0 is done 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 collect(): s=0x5636c4d3ab70 stack=0x5636c48737c0 is complete 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 10 min_generation is 0 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 10 trimming bilogs for bucket=:dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) markers=[,00000000002.6.6,,00000000001.11.6,,00000000001.7.6,00000000002.9.6,00000000001.8.6,00000000001.12.6,00000000001.7.6,], shards=11 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d4fbc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 10 trimming bilog shard 1 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000002.6.6 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 10 trimming bilog shard 3 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.11.6 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 10 trimming bilog shard 5 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.7.6 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 10 trimming bilog shard 6 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000002.9.6 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 10 trimming bilog shard 7 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.8.6 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 10 trimming bilog shard 8 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.12.6 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 10 trimming bilog shard 9 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.7.6 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4ceb000:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d546c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d59690:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4ceba10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d49d90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4cd8120:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 cr:s=0x5636c4d49930:op=0x5636c4d4c740:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4ceb000:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.1 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 run: stack=0x5636c4884340 is io blocked 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d546c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.3 2026-03-20T11:46:49.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is io blocked 2026-03-20T11:46:49.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d59690:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.5 2026-03-20T11:46:49.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 run: stack=0x5636c4ce56a0 is io blocked 2026-03-20T11:46:49.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4ceba10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.6 2026-03-20T11:46:49.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 run: stack=0x5636c4886580 is io blocked 2026-03-20T11:46:49.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d49d90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.7 2026-03-20T11:46:49.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 run: stack=0x5636c4cee790 is io blocked 2026-03-20T11:46:49.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4cd8120:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.8 2026-03-20T11:46:49.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 run: stack=0x5636c4873b00 is io blocked 2026-03-20T11:46:49.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 cr:s=0x5636c4d49930:op=0x5636c4d4c740:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.9 2026-03-20T11:46:49.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.154+0000 7f25a9678900 20 run: stack=0x5636c4d49930 is io blocked 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d49d90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 cr:s=0x5636c4cee790:op=0x5636c4d49d90:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 15 stack 0x5636c4cee790 end 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 run: stack=0x5636c4cee790 is done 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d4fbc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d59690:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 cr:s=0x5636c4ce56a0:op=0x5636c4d59690:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 15 stack 0x5636c4ce56a0 end 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 run: stack=0x5636c4ce56a0 is done 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d546c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 cr:s=0x5636c4d32ac0:op=0x5636c4d546c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 15 stack 0x5636c4d32ac0 end 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 run: stack=0x5636c4d32ac0 is done 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4cd8120:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 cr:s=0x5636c4873b00:op=0x5636c4cd8120:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 15 stack 0x5636c4873b00 end 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 run: stack=0x5636c4873b00 is done 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d4fbc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d4fbc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d4fbc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4ceb000:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 cr:s=0x5636c4884340:op=0x5636c4ceb000:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 15 stack 0x5636c4884340 end 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 run: stack=0x5636c4884340 is done 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d4fbc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4ceba10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 cr:s=0x5636c4886580:op=0x5636c4ceba10:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 15 stack 0x5636c4886580 end 2026-03-20T11:46:49.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 run: stack=0x5636c4886580 is done 2026-03-20T11:46:49.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d4fbc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.156+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.157+0000 7f25a9678900 20 cr:s=0x5636c4d49930:op=0x5636c4d4c740:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.157+0000 7f25a9678900 20 cr:s=0x5636c4d49930:op=0x5636c4d4c740:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.157+0000 7f25a9678900 15 stack 0x5636c4d49930 end 2026-03-20T11:46:49.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.157+0000 7f25a9678900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.157+0000 7f25a9678900 20 run: stack=0x5636c4d49930 is done 2026-03-20T11:46:49.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.157+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4d4fbc0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.157+0000 7f25a9678900 20 cr:s=0x5636c4d3ab70:op=0x5636c4cd0d40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.157+0000 7f25a9678900 20 trim: trimmed bucket instance dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:46:49.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.157+0000 7f25a9678900 15 stack 0x5636c4d3ab70 end 2026-03-20T11:46:49.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.157+0000 7f25a9678900 20 run: stack=0x5636c4d3ab70 is done 2026-03-20T11:46:49.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.157+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:49.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.157+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccf460:27BucketTrimInstanceCollectCR: operate() returned r=-22 2026-03-20T11:46:49.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.157+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccb620:12BucketTrimCR: operate() 2026-03-20T11:46:49.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.157+0000 7f25a9678900 20 writing bucket trim marker=16:69e609d8:root::dipepu-52:head 2026-03-20T11:46:49.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.157+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4d56980:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:49.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.157+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4d56980:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:49.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.157+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is io blocked 2026-03-20T11:46:49.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.158+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4d56980:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:49.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.158+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4d56980:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:49.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.158+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4d56980:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:49.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.158+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4d56980:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:49.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.158+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccb620:12BucketTrimCR: operate() 2026-03-20T11:46:49.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.158+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4d56980:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:49.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.158+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4d56980:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:49.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.158+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is io blocked 2026-03-20T11:46:49.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.158+0000 7f257e7fc640 20 trim: bucket trim completed 2026-03-20T11:46:49.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.159+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4d56980:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:49.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.159+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4d56980:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:49.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.159+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4d56980:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:49.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.159+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4d56980:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:49.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.159+0000 7f25a9678900 20 cr:s=0x5636c4ccbd20:op=0x5636c4ccb620:12BucketTrimCR: operate() 2026-03-20T11:46:49.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.159+0000 7f25a9678900 4 bucket index log processing completed in 0.720897734s 2026-03-20T11:46:49.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.159+0000 7f25a9678900 15 stack 0x5636c4ccbd20 end 2026-03-20T11:46:49.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.159+0000 7f25a9678900 20 run: stack=0x5636c4ccbd20 is done 2026-03-20T11:46:49.162 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.165+0000 7f25a9678900 20 remove_watcher() i=6 2026-03-20T11:46:49.162 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.165+0000 7f25a9678900 2 removed watcher, disabling cache 2026-03-20T11:46:49.163 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.166+0000 7f25a9678900 20 remove_watcher() i=1 2026-03-20T11:46:49.163 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.166+0000 7f25a9678900 20 remove_watcher() i=3 2026-03-20T11:46:49.163 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.166+0000 7f25a9678900 20 remove_watcher() i=4 2026-03-20T11:46:49.163 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.166+0000 7f25a9678900 20 remove_watcher() i=7 2026-03-20T11:46:49.163 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.166+0000 7f25a9678900 20 remove_watcher() i=2 2026-03-20T11:46:49.163 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.166+0000 7f25a9678900 20 remove_watcher() i=5 2026-03-20T11:46:49.163 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.166+0000 7f25a9678900 20 remove_watcher() i=0 2026-03-20T11:46:49.220 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:49.220 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:49.220 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:46:49.258 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:49.258 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:49.349 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:46:49.362 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:49.364+0000 7fcfb29b3640 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2026-03-20T11:46:49.362 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:49.364+0000 7fcfb59b9640 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2026-03-20T11:46:49.362 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:49.364+0000 7fcfaf9ad640 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2026-03-20T11:46:49.363 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774007137.857782_141.1","last_update":"2026-03-20T11:45:37.857782Z"},{"marker":"1_1774007137.912792_174.1","last_update":"2026-03-20T11:45:37.912792Z"},{"marker":"1_1774007196.520119_280.1","last_update":"2026-03-20T11:46:36.520119Z"},{"marker":"1_1774007039.336409_210.1","last_update":"2026-03-20T11:43:59.336409Z"}] 2026-03-20T11:46:49.364 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:46:49.364 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:49.364 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:46:49.364 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:46:49.449 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:49.449 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:49.531 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.534+0000 7f651fc1c900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:46:49.532 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.535+0000 7f651fc1c900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:46:49.545 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774007137.857782_141.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:45:37.857782Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774007137.912792_174.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:37.912792Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774007196.520119_280.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:46:36.520119Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774007039.336409_210.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:59.336409Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:46:49Z"} 2026-03-20T11:46:49.545 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:46:49.545 INFO:rgw_multi.tests:trimming on zone=a1 2026-03-20T11:46:49.545 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '50', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:46:49.545 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '50', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:46:49.545 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bilog autotrim --rgw-sync-log-trim-max-buckets 50 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 20 --debug-ms 0 2026-03-20T11:46:49.582 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:49.582 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:49.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.601+0000 7fdbacf23900 20 rados->read ofs=0 len=0 2026-03-20T11:46:49.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.602+0000 7fdbacf23900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:46:49.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.602+0000 7fdbacf23900 20 rados->read ofs=0 len=0 2026-03-20T11:46:49.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.602+0000 7fdbacf23900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:46:49.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.602+0000 7fdbacf23900 20 realm test-realm 6a789b89-c2b8-45f5-82cc-fb80234d0c6c 2026-03-20T11:46:49.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.602+0000 7fdbacf23900 20 rados->read ofs=0 len=0 2026-03-20T11:46:49.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.603+0000 7fdbacf23900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:46:49.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.603+0000 7fdbacf23900 20 rados->read ofs=0 len=0 2026-03-20T11:46:49.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.603+0000 7fdbacf23900 20 rados_obj.operate() r=0 bl.length=10 2026-03-20T11:46:49.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.603+0000 7fdbacf23900 20 rados->read ofs=0 len=0 2026-03-20T11:46:49.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.604+0000 7fdbacf23900 20 rados_obj.operate() r=0 bl.length=2020 2026-03-20T11:46:49.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.604+0000 7fdbacf23900 20 rados->read ofs=0 len=0 2026-03-20T11:46:49.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.604+0000 7fdbacf23900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:46:49.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.604+0000 7fdbacf23900 20 rados->read ofs=0 len=0 2026-03-20T11:46:49.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.604+0000 7fdbacf23900 20 rados_obj.operate() r=0 bl.length=1030 2026-03-20T11:46:49.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.604+0000 7fdbacf23900 20 zone a1 found 2026-03-20T11:46:49.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.604+0000 7fdbacf23900 4 Realm: test-realm (6a789b89-c2b8-45f5-82cc-fb80234d0c6c) 2026-03-20T11:46:49.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.604+0000 7fdbacf23900 4 ZoneGroup: a (a) 2026-03-20T11:46:49.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.604+0000 7fdbacf23900 4 Zone: a1 (4b634ebf-58f5-45bc-839a-cb4b425e773a) 2026-03-20T11:46:49.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.604+0000 7fdbacf23900 4 using period configuration: cb2fcdce-2c1b-429a-8601-a284ee3a2381:12 2026-03-20T11:46:49.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.604+0000 7fdbacf23900 20 period zonegroup name a 2026-03-20T11:46:49.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.604+0000 7fdbacf23900 20 using current period zonegroup a 2026-03-20T11:46:49.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.604+0000 7fdbacf23900 20 rados->read ofs=0 len=0 2026-03-20T11:46:49.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.605+0000 7fdbacf23900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.605+0000 7fdbacf23900 20 rados->read ofs=0 len=0 2026-03-20T11:46:49.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.605+0000 7fdbacf23900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.605+0000 7fdbacf23900 20 init(): pipe_map (zone=4b634ebf-58f5-45bc-839a-cb4b425e773a bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:46:49.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.605+0000 7fdbacf23900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.605+0000 7fdbacf23900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.605+0000 7fdbacf23900 20 rados->read ofs=0 len=0 2026-03-20T11:46:49.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.606+0000 7fdbacf23900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.606+0000 7fdbacf23900 20 rados->read ofs=0 len=0 2026-03-20T11:46:49.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.606+0000 7fdbacf23900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.606+0000 7fdbacf23900 20 init(): pipe_map (zone=51055576-6662-409d-8abb-ecaeefd3dd80 bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:46:49.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.606+0000 7fdbacf23900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.606+0000 7fdbacf23900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.606+0000 7fdbacf23900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.606+0000 7fdbacf23900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.606+0000 7fdbacf23900 20 started sync module instance, tier type = 2026-03-20T11:46:49.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.606+0000 7fdbacf23900 20 generating connection object for zone a2 id 51055576-6662-409d-8abb-ecaeefd3dd80 2026-03-20T11:46:49.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.606+0000 7fdbacf23900 20 started zone id=4b634ebf-58f5-45bc-839a-cb4b425e773a (name=a1) with tier type = 2026-03-20T11:46:49.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.610+0000 7fdbacf23900 20 add_watcher() i=5 2026-03-20T11:46:49.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.610+0000 7fdbacf23900 20 add_watcher() i=6 2026-03-20T11:46:49.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.610+0000 7fdbacf23900 20 add_watcher() i=1 2026-03-20T11:46:49.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.610+0000 7fdbacf23900 20 add_watcher() i=4 2026-03-20T11:46:49.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.611+0000 7fdbacf23900 20 add_watcher() i=2 2026-03-20T11:46:49.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.611+0000 7fdbacf23900 20 add_watcher() i=0 2026-03-20T11:46:49.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.611+0000 7fdbacf23900 20 add_watcher() i=7 2026-03-20T11:46:49.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.611+0000 7fdbacf23900 20 add_watcher() i=3 2026-03-20T11:46:49.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.611+0000 7fdbacf23900 2 all 8 watchers are set, enabling cache 2026-03-20T11:46:49.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.613+0000 7fdbacf23900 20 rgw_check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2026-03-20T11:46:49.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.613+0000 7fdbacf23900 20 rgw_check_secure_mon_conn(): mode 1 is insecure 2026-03-20T11:46:49.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.613+0000 7fdbacf23900 5 note: GC not initialized 2026-03-20T11:46:49.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.613+0000 7fdb567e4640 20 reqs_thread_entry: start 2026-03-20T11:46:49.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.669+0000 7fdbacf23900 20 init_complete bucket index max shards: 11 2026-03-20T11:46:49.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.669+0000 7fdbacf23900 20 Filter name: none 2026-03-20T11:46:49.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.669+0000 7fdb07fff640 20 reqs_thread_entry: start 2026-03-20T11:46:49.666 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.669+0000 7fdb067fc640 20 reqs_thread_entry: start 2026-03-20T11:46:49.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.671+0000 7fdbacf23900 10 rgw bucket trim manager: Watching bilog.trim 2026-03-20T11:46:49.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.671+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bc300:12BucketTrimCR: operate() 2026-03-20T11:46:49.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.671+0000 7fdbacf23900 10 fetching active bucket counters 2026-03-20T11:46:49.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.671+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bea20:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:49.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.671+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bea20:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:49.668 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.671+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is io blocked 2026-03-20T11:46:49.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.672+0000 7fdb77fff640 20 trim: get_bucket_counters: [] 2026-03-20T11:46:49.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.672+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bea20:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:49.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.672+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bea20:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:49.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.672+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bea20:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:49.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.672+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bea20:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:49.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.672+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bc300:12BucketTrimCR: operate() 2026-03-20T11:46:49.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.672+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:49.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.672+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:49.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.672+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is io blocked 2026-03-20T11:46:49.669 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.673+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:49.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.673+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:49.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.673+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:49.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.673+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:49.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.673+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bc300:12BucketTrimCR: operate() 2026-03-20T11:46:49.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.673+0000 7fdbacf23900 10 listing cold buckets from marker=12:69e609d8:root::dipepu-52:head 2026-03-20T11:46:49.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.673+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:14MetadataListCR: operate() 2026-03-20T11:46:49.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.673+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:14MetadataListCR: operate() 2026-03-20T11:46:49.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.673+0000 7fdbacf23900 20 enqueued request req=0x7fdb64002c90 2026-03-20T11:46:49.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.673+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:49.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.673+0000 7fdbacf23900 20 req: 0x7fdb64002c90 2026-03-20T11:46:49.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.673+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is io blocked 2026-03-20T11:46:49.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.673+0000 7fdb57fe7640 20 dequeued request req=0x7fdb64002c90 2026-03-20T11:46:49.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.673+0000 7fdb57fe7640 20 RGWWQ: empty 2026-03-20T11:46:49.670 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.673+0000 7fdb57fe7640 20 rgw async rados processor: starting metadata listing at 12:69e609d8:root::dipepu-52:head 2026-03-20T11:46:49.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.678+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-52 2026-03-20T11:46:49.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.678+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-32 2026-03-20T11:46:49.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.678+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-42 2026-03-20T11:46:49.675 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.678+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:46:49.679 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.682+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:46:49.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.686+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-7 2026-03-20T11:46:49.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.686+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-54 2026-03-20T11:46:49.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.686+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-49 2026-03-20T11:46:49.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.686+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-47 2026-03-20T11:46:49.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.686+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:46:49.688 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.691+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:46:49.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.695+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-11 2026-03-20T11:46:49.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.695+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-12 2026-03-20T11:46:49.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.695+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-41 2026-03-20T11:46:49.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.695+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-23 2026-03-20T11:46:49.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.695+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-15 2026-03-20T11:46:49.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.695+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-45 2026-03-20T11:46:49.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.695+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:46:49.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.699+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:46:49.699 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.702+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-9 2026-03-20T11:46:49.699 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.702+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:46:49.702 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.705+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-29 2026-03-20T11:46:49.702 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.705+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:46:49.705 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.708+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-31 2026-03-20T11:46:49.705 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.708+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:46:49.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.711+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-33 2026-03-20T11:46:49.708 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.711+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:49.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.714+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-4 2026-03-20T11:46:49.711 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.714+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:46:49.713 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.716+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:49.716 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.719+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:49.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.722+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-43 2026-03-20T11:46:49.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.722+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-30 2026-03-20T11:46:49.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.722+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-18 2026-03-20T11:46:49.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.722+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-2 2026-03-20T11:46:49.719 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.722+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:46:49.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.724+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-8 2026-03-20T11:46:49.721 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.724+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:46:49.723 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.726+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:49.725 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.728+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-58 2026-03-20T11:46:49.725 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.728+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:46:49.727 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.730+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:49.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.731+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-27 2026-03-20T11:46:49.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.732+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-28 2026-03-20T11:46:49.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.732+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-59 2026-03-20T11:46:49.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.732+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:46:49.730 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.733+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:49.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.734+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-40 2026-03-20T11:46:49.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.734+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-22 2026-03-20T11:46:49.731 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.735+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:46:49.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.736+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-56 2026-03-20T11:46:49.733 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.736+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:46:49.734 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.737+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:46:49.735 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.738+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:49.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.739+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-53 2026-03-20T11:46:49.736 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.739+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:46:49.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.740+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:46:49.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.741+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:46:49.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.742+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:46:49.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.743+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:46:49.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.744+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-34 2026-03-20T11:46:49.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.744+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-21 2026-03-20T11:46:49.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.744+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-14 2026-03-20T11:46:49.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.744+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-50 2026-03-20T11:46:49.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.744+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-57 2026-03-20T11:46:49.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.744+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:46:49.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.744+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-39 2026-03-20T11:46:49.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.744+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:49.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.745+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-37 2026-03-20T11:46:49.742 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.745+0000 7fdb57fe7640 20 rgw async rados processor: restarting metadata listing 2026-03-20T11:46:49.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.752+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-10 2026-03-20T11:46:49.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.752+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:49.756 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.759+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:49.765 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.768+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:49.772 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.775+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:49.779 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.782+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:49.786 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.789+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:49.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.795+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-19 2026-03-20T11:46:49.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.795+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-3 2026-03-20T11:46:49.792 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.795+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:49.799 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.802+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:49.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.809+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-17 2026-03-20T11:46:49.806 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.809+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:49.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.815+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-38 2026-03-20T11:46:49.812 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.815+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:49.819 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.822+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-24 2026-03-20T11:46:49.819 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.822+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-36 2026-03-20T11:46:49.819 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.822+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:49.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.829+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:49.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.835+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:49.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.842+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-44 2026-03-20T11:46:49.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.842+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-35 2026-03-20T11:46:49.839 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.842+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:49.845 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.848+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-16 2026-03-20T11:46:49.846 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.848+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:49.852 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.855+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:49.859 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.861+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:49.866 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.869+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-55 2026-03-20T11:46:49.866 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.869+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-13 2026-03-20T11:46:49.866 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.869+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-48 2026-03-20T11:46:49.866 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.869+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-25 2026-03-20T11:46:49.866 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.869+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-51 2026-03-20T11:46:49.866 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.869+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:49.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdb57fe7640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:49.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:14MetadataListCR: operate() 2026-03-20T11:46:49.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:14MetadataListCR: operate() 2026-03-20T11:46:49.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:14MetadataListCR: operate() 2026-03-20T11:46:49.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:14MetadataListCR: operate() 2026-03-20T11:46:49.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bc300:12BucketTrimCR: operate() 2026-03-20T11:46:49.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 4 collected 50 buckets for trim 2026-03-20T11:46:49.873 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 4 starting trim on bucket=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 4 starting trim on bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 cr:s=0x5608c11bece0:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 4 starting trim on bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 4 starting trim on bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11c4a80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c51d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 cr:s=0x5608c11bece0:op=0x5608c11c5920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c6070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11c4a80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 enqueued request req=0x5608c11c6a20 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 req: 0x5608c11c6a20 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is io blocked 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c51d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 enqueued request req=0x5608c11c6dc0 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 req: 0x5608c11c6a20 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 req: 0x5608c11c6dc0 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 run: stack=0x5608c11fb6f0 is io blocked 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 cr:s=0x5608c11bece0:op=0x5608c11c5920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 enqueued request req=0x5608c11c7160 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 req: 0x5608c11c6a20 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 req: 0x5608c11c6dc0 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 req: 0x5608c11c7160 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 run: stack=0x5608c11bece0 is io blocked 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c6070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 enqueued request req=0x5608c11c7580 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 req: 0x5608c11c6a20 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 req: 0x5608c11c6dc0 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 req: 0x5608c11c7160 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 req: 0x5608c11c7580 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdbacf23900 20 run: stack=0x5608c11c0950 is io blocked 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdb6a7fc640 20 dequeued request req=0x5608c11c6a20 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdb6a7fc640 20 RGWWQ: 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdb6a7fc640 20 req: 0x5608c11c6dc0 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdb6a7fc640 20 req: 0x5608c11c7160 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdb6a7fc640 20 req: 0x5608c11c7580 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdb6a7fc640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 : miss 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdb6a7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdb69ffb640 20 dequeued request req=0x5608c11c6dc0 2026-03-20T11:46:49.874 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdb69ffb640 20 RGWWQ: 2026-03-20T11:46:49.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdb69ffb640 20 req: 0x5608c11c7160 2026-03-20T11:46:49.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdb69ffb640 20 req: 0x5608c11c7580 2026-03-20T11:46:49.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdb69ffb640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 : miss 2026-03-20T11:46:49.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdb69ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdb6affd640 20 dequeued request req=0x5608c11c7160 2026-03-20T11:46:49.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdb6affd640 20 RGWWQ: 2026-03-20T11:46:49.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdb6affd640 20 req: 0x5608c11c7580 2026-03-20T11:46:49.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdb6affd640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 : miss 2026-03-20T11:46:49.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdb6affd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdb6b7fe640 20 dequeued request req=0x5608c11c7580 2026-03-20T11:46:49.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdb6b7fe640 20 RGWWQ: empty 2026-03-20T11:46:49.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdb6b7fe640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 : miss 2026-03-20T11:46:49.875 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.876+0000 7fdb6b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6affd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:49.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6affd640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 info.flags=0x17 2026-03-20T11:46:49.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6affd640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 to cache LRU end 2026-03-20T11:46:49.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6affd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:49.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6affd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:49.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6affd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:46:49.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6affd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-59 : miss 2026-03-20T11:46:49.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6affd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb69ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:49.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb69ffb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 info.flags=0x17 2026-03-20T11:46:49.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb69ffb640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 to cache LRU end 2026-03-20T11:46:49.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb69ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:49.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb69ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:49.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb69ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:46:49.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb69ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:46:49.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb69ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-48 : miss 2026-03-20T11:46:49.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb69ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6a7fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=437 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6a7fc640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 info.flags=0x17 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6a7fc640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 to cache LRU end 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6a7fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6a7fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6a7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6a7fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-61 : miss 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6a7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6affd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6affd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-59 info.flags=0x0 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6affd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-59 to cache LRU end 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6affd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-59 : miss 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6affd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb69ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb69ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-48 info.flags=0x0 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb69ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-48 to cache LRU end 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb69ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-48 : miss 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb69ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6a7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6a7fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-61 info.flags=0x0 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6a7fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-61 to cache LRU end 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6a7fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-61 : miss 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6a7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6b7fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6b7fe640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 info.flags=0x17 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6b7fe640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 to cache LRU end 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6b7fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6b7fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6b7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6b7fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-21 : miss 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.877+0000 7fdb6b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6affd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6affd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-59 info.flags=0x0 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6affd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-59 to cache LRU end 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6affd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19): adding source pipe: {s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6affd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19): adding dest pipe: {s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6affd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6affd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6affd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6a7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6a7fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-61 info.flags=0x0 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6a7fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-61 to cache LRU end 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 cr:s=0x5608c11bece0:op=0x5608c11c5920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 cr:s=0x5608c11bece0:op=0x5608c11c5920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21): adding source pipe: {s={b=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 cr:s=0x5608c11bece0:op=0x5608c11c5920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 cr:s=0x5608c11bece0:op=0x5608c11c5920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21): adding dest pipe: {s={b=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 cr:s=0x5608c11bece0:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c0d599b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 cr:s=0x5608c11bece0:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 run: stack=0x5608c11bece0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c0d59fd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c0d59fd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6a7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:46:49.877 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6b7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6b7fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-21 info.flags=0x0 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6b7fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-21 to cache LRU end 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6b7fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-21 : miss 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb69ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb69ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-48 info.flags=0x0 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb69ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-48 to cache LRU end 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb69ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8): adding source pipe: {s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb69ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8): adding dest pipe: {s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb69ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb69ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 9aa4c6e36ccb6233762d082a38474580502587a0d99e65fde57881297415866c 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb69ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114649Z 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:9aa4c6e36ccb6233762d082a38474580502587a0d99e65fde57881297415866c 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = a78330f954574c3c9df7ddc6e242d10967a7beb7c74636490b17e7c940323169 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a78330f954574c3c9df7ddc6e242d10967a7beb7c74636490b17e7c940323169 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114649Z 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=0, curl_handle=0x5608c0d5bb20 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is io blocked 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11c4a80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c51d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb067fc640 20 link_request req_data=0x5608c0d5c620 req_data->id=0, curl_handle=0x5608c0d5bb20 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11c4a80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c51d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11c4a80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c51d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11c4a80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c51d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6b7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6b7fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-21 info.flags=0x0 2026-03-20T11:46:49.878 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6b7fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-21 to cache LRU end 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26): adding source pipe: {s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26): adding dest pipe: {s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c11c4f30:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c0d5e2b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 run: stack=0x5608c11fb6f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c0d5e8d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c0d5ef00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c0d5e8d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb6b7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 0173740fe97ba2fa649c12b1fad7fe71b60781c78e1cd25d5f6d226461882960 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114649Z 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:0173740fe97ba2fa649c12b1fad7fe71b60781c78e1cd25d5f6d226461882960 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 77ec212f3b42ed28b147303dc011bc5266348f0c14342240a06b7148df4ce4fb 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=77ec212f3b42ed28b147303dc011bc5266348f0c14342240a06b7148df4ce4fb 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114649Z 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=1, curl_handle=0x5608c0d60ac0 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 run: stack=0x5608c11c55c0 is io blocked 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c0d5ef00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdb067fc640 20 link_request req_data=0x5608c0d61210 req_data->id=1, curl_handle=0x5608c0d60ac0 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.878+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.879 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 69ded0d33b69fdbda9f5de78c540e8bb9741ce66b32d520717b4a3ad3e0c12fb 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114649Z 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:69ded0d33b69fdbda9f5de78c540e8bb9741ce66b32d520717b4a3ad3e0c12fb 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 199808012da8266b559cd1d92e81ab3eabecae956e2644a058532712f5887ec4 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=199808012da8266b559cd1d92e81ab3eabecae956e2644a058532712f5887ec4 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114649Z 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=2, curl_handle=0x5608c11c9480 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdb067fc640 20 link_request req_data=0x5608c11c9b80 req_data->id=2, curl_handle=0x5608c11c9480 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is io blocked 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c6070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c6070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c6070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c6070:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 20 cr:s=0x5608c11cbb70:op=0x5608c11cb550:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 20 run: stack=0x5608c11c0950 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 20 cr:s=0x5608c11cbb70:op=0x5608c11cbd00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 20 cr:s=0x5608c11cbb70:op=0x5608c11cbd00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:49.880 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:49.881 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.881 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.881 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.881 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.881 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.881 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:49.881 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 90a3e6a9561b98af4010301e1c392c6675cdf937f175a8510ec7b83e5045ac09 2026-03-20T11:46:49.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:49.881 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114649Z 2026-03-20T11:46:49.881 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:49.881 INFO:teuthology.orchestra.run.vm02.stderr:90a3e6a9561b98af4010301e1c392c6675cdf937f175a8510ec7b83e5045ac09 2026-03-20T11:46:49.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:49.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:49.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:49.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:49.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = bba134ba1ffa47d321732b7252bad935bb8a3d3a8ea5ba8fb6f91d3e15d72ee4 2026-03-20T11:46:49.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=bba134ba1ffa47d321732b7252bad935bb8a3d3a8ea5ba8fb6f91d3e15d72ee4 2026-03-20T11:46:49.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114649Z 2026-03-20T11:46:49.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:49.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=3, curl_handle=0x5608c11cd8e0 2026-03-20T11:46:49.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdbacf23900 20 run: stack=0x5608c11cbb70 is io blocked 2026-03-20T11:46:49.881 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.879+0000 7fdb067fc640 20 link_request req_data=0x5608c11cdff0 req_data->id=3, curl_handle=0x5608c11cd8e0 2026-03-20T11:46:49.881 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:49.879+0000 7fd0b8bbf640 -1 req 10297786386525814614 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:49.881 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:49.882+0000 7fd0b1bb1640 -1 req 11257200609451215594 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:49.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.880+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c0d5ef00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.880+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c0d5ef00:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:49.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.880+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c0d5e2b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.880+0000 7fdbacf23900 15 stack 0x5608c11c4c90 end 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.880+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is done 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.880+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.880+0000 7fdbacf23900 20 collect(): s=0x5608c11fb6f0 stack=0x5608c11c4c90 is complete 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.880+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.880+0000 7fdbacf23900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.880+0000 7fdbacf23900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.880+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.880+0000 7fdbacf23900 15 stack 0x5608c11fb6f0 end 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.880+0000 7fdbacf23900 20 stack->operate() returned ret=-22 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.880+0000 7fdbacf23900 20 run: stack=0x5608c11fb6f0 is done 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.880+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.880+0000 7fdbacf23900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.880+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.880+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.880+0000 7fdbacf23900 4 starting trim on bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.880+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c0d614f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.880+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c0d614f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.880+0000 7fdbacf23900 20 enqueued request req=0x5608c11c6a20 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.880+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.880+0000 7fdbacf23900 20 req: 0x5608c11c6a20 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.880+0000 7fdbacf23900 20 run: stack=0x5608c11fb6f0 is io blocked 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.881+0000 7fdb697fa640 20 dequeued request req=0x5608c11c6a20 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.881+0000 7fdb697fa640 20 RGWWQ: empty 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.881+0000 7fdb697fa640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 : miss 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.881+0000 7fdb697fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.881+0000 7fdb697fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.881+0000 7fdb697fa640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 info.flags=0x17 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.881+0000 7fdb697fa640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 to cache LRU end 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.881+0000 7fdb697fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.881+0000 7fdb697fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.881+0000 7fdb697fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.881+0000 7fdb697fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-41 : miss 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.881+0000 7fdb697fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdb697fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdb697fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-41 info.flags=0x0 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdb697fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-41 to cache LRU end 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdb697fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-41 : miss 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdb697fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdb697fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdb697fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-41 info.flags=0x0 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdb697fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-41 to cache LRU end 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdb697fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1): adding source pipe: {s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdb697fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1): adding dest pipe: {s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdb697fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdb697fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdb697fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c0d614f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c0d614f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c0d614f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c0d614f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c0d5ef00:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 20 run: stack=0x5608c11fb6f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c0d614f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c0d614f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 6436a899ee59b039676ecbadfafffa5a464be8f71e673261d22cc2909cbf8091 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114649Z 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:6436a899ee59b039676ecbadfafffa5a464be8f71e673261d22cc2909cbf8091 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = a8588d2182cea2bc83aa45fb2752f34bb035b02aeb0dc85552afb09298df8a30 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a8588d2182cea2bc83aa45fb2752f34bb035b02aeb0dc85552afb09298df8a30 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114649Z 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=4, curl_handle=0x5608c11c9480 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is io blocked 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.882+0000 7fdb067fc640 20 link_request req_data=0x5608c11c9b80 req_data->id=4, curl_handle=0x5608c11c9480 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 cr:s=0x5608c11cbb70:op=0x5608c11cbd00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 cr:s=0x5608c11cbb70:op=0x5608c11cbd00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c0d59fd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.887 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c0d5e8d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 cr:s=0x5608c11cbb70:op=0x5608c11cbd00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c0d59fd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c0d5e8d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 cr:s=0x5608c11cbb70:op=0x5608c11cbd00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c0d59fd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c0d5e8d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 cr:s=0x5608c11cbb70:op=0x5608c11cb550:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 15 stack 0x5608c11cbb70 end 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 run: stack=0x5608c11cbb70 is done 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c0d59fd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c0d5e8d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 collect(): s=0x5608c11c0950 stack=0x5608c11cbb70 is complete 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 10 trimming bilogs for bucket=:dipepu-21[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26]) markers=[,,,,,,,,00000000001.60.14,,], shards=11 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c0d599b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 15 stack 0x5608c11c5dd0 end 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is done 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c11c4f30:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 15 stack 0x5608c11c55c0 end 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 run: stack=0x5608c11c55c0 is done 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c0d5b190:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 10 trimming bilog shard 8 of :dipepu-21[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26]) at marker 00000000001.60.14 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 run: stack=0x5608c11c0950 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 cr:s=0x5608c11bece0:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 collect(): s=0x5608c11bece0 stack=0x5608c11c5dd0 is complete 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 10 trimming bilogs for bucket=:dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) markers=[,00000000001.44.6,,,,00000000001.24.6,,,00000000001.44.6,,00000000001.39.6], shards=11 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 collect(): s=0x5608c10a18e0 stack=0x5608c11c55c0 is complete 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 10 min_generation is 1 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 cr:s=0x5608c11cbb70:op=0x5608c0d5f530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 cr:s=0x5608c11bece0:op=0x5608c11cb550:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 10 trimming bilog shard 1 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.44.6 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 10 trimming bilog shard 5 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.24.6 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 10 trimming bilog shard 8 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.44.6 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 10 trimming bilog shard 10 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.39.6 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 run: stack=0x5608c11bece0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c0d5e8d0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.0 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.1 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.2 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.3 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.4 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.5 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.6 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.7 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.8 2026-03-20T11:46:49.888 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.9 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.10 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 cr:s=0x5608c11cbb70:op=0x5608c0d5f530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.883+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26.8 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 run: stack=0x5608c11cbb70 is io blocked 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c12082f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11d5690:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c0d6eed0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c0d6e300:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1204360:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c4f30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 cr:s=0x5608c11c97c0:op=0x5608c1208d20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d6cbe0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 cr:s=0x5608c0d5be40:op=0x5608c0d6d1d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 cr:s=0x5608c0d61030:op=0x5608c12099d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 cr:s=0x5608c0d5c440:op=0x5608c1209fc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 cr:s=0x5608c1207b70:op=0x5608c11cbcb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 cr:s=0x5608c0d60240:op=0x5608c11cc2a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 cr:s=0x5608c0d603c0:op=0x5608c1201400:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 cr:s=0x5608c11cd220:op=0x5608c12019f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c12082f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.1 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 run: stack=0x5608c11c55c0 is io blocked 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11d5690:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.5 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is io blocked 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c0d6eed0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.8 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is io blocked 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c0d6e300:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.10 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is io blocked 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1204360:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is io blocked 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c4f30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 run: stack=0x5608c0d61b20 is io blocked 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 cr:s=0x5608c11c97c0:op=0x5608c1208d20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 run: stack=0x5608c11c97c0 is io blocked 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d6cbe0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 run: stack=0x5608c11d3560 is io blocked 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.884+0000 7fdbacf23900 20 cr:s=0x5608c0d5be40:op=0x5608c0d6d1d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.885+0000 7fdbacf23900 20 run: stack=0x5608c0d5be40 is io blocked 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.885+0000 7fdbacf23900 20 cr:s=0x5608c0d61030:op=0x5608c12099d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 run: stack=0x5608c0d61030 is io blocked 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c0d5c440:op=0x5608c1209fc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 run: stack=0x5608c0d5c440 is io blocked 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c1207b70:op=0x5608c11cbcb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 run: stack=0x5608c1207b70 is io blocked 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c0d60240:op=0x5608c11cc2a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 run: stack=0x5608c0d60240 is io blocked 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c0d603c0:op=0x5608c1201400:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 run: stack=0x5608c0d603c0 is io blocked 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11cd220:op=0x5608c12019f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 run: stack=0x5608c11cd220 is io blocked 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11d5690:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11d5690:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 15 stack 0x5608c11c5dd0 end 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is done 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d6cbe0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c4f30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c12082f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c12082f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 15 stack 0x5608c11c55c0 end 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 run: stack=0x5608c11c55c0 is done 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c0d6eed0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c0d6eed0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 15 stack 0x5608c0d6dec0 end 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is done 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11c97c0:op=0x5608c1208d20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1204360:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c0d5be40:op=0x5608c0d6d1d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c0d6e300:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c0d6e300:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 15 stack 0x5608c11c5f10 end 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is done 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11cbb70:op=0x5608c0d5f530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11cbb70:op=0x5608c0d5f530:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 15 stack 0x5608c11cbb70 end 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 run: stack=0x5608c11cbb70 is done 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11bece0:op=0x5608c11cb550:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d6cbe0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c4f30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11c97c0:op=0x5608c1208d20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1204360:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c0d5be40:op=0x5608c0d6d1d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c0d5b190:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11bece0:op=0x5608c11cb550:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d6cbe0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c4f30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11c97c0:op=0x5608c1208d20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1204360:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c0d5be40:op=0x5608c0d6d1d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 trim: trimmed bucket instance dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 15 stack 0x5608c11c0950 end 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 run: stack=0x5608c11c0950 is done 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11bece0:op=0x5608c11cb550:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d6cbe0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 15 stack 0x5608c11d3560 end 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 run: stack=0x5608c11d3560 is done 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c4f30:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 15 stack 0x5608c0d61b20 end 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 run: stack=0x5608c0d61b20 is done 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11c97c0:op=0x5608c1208d20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 15 stack 0x5608c11c97c0 end 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 run: stack=0x5608c11c97c0 is done 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1204360:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 15 stack 0x5608c11d3250 end 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is done 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c0d5be40:op=0x5608c0d6d1d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.890 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 15 stack 0x5608c0d5be40 end 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 run: stack=0x5608c0d5be40 is done 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11bece0:op=0x5608c11cb550:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c0d5e8d0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.887+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdbacf23900 4 starting trim on bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdbacf23900 20 cr:s=0x5608c11bece0:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdbacf23900 20 trim: trimmed bucket instance dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdbacf23900 15 stack 0x5608c11bece0 end 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdbacf23900 20 run: stack=0x5608c11bece0 is done 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c0d5e8d0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c0d6eed0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c0d5e8d0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c0d6eed0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdbacf23900 20 enqueued request req=0x5608c11bee80 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdbacf23900 20 req: 0x5608c11bee80 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdbacf23900 20 run: stack=0x5608c11c0950 is io blocked 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdbacf23900 4 starting trim on bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c0d5e8d0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c0d5f530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c0d5e8d0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c0d5f530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdb68ff9640 20 dequeued request req=0x5608c11bee80 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdb68ff9640 20 RGWWQ: empty 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdb68ff9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 : miss 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdb68ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdbacf23900 20 enqueued request req=0x5608c11bff40 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdbacf23900 20 req: 0x5608c11bff40 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdbacf23900 20 run: stack=0x5608c0d61b20 is io blocked 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdb68ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdb68ff9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 info.flags=0x17 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdb68ff9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 to cache LRU end 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdb68ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdb68ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdb68ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdb68ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-49 : miss 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdb68ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdb63fff640 20 dequeued request req=0x5608c11bff40 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdb63fff640 20 RGWWQ: empty 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdb63fff640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 : miss 2026-03-20T11:46:49.891 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.888+0000 7fdb63fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.892 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:49.890+0000 7fd0bd3c8640 -1 req 9033116652295221969 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:49.892 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:49.893+0000 7fcfe2212640 -1 req 13719275396551742011 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:49.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb68ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb68ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-49 info.flags=0x0 2026-03-20T11:46:49.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb68ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-49 to cache LRU end 2026-03-20T11:46:49.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb68ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-49 : miss 2026-03-20T11:46:49.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb68ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb63fff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:49.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb63fff640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 info.flags=0x17 2026-03-20T11:46:49.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb63fff640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 to cache LRU end 2026-03-20T11:46:49.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb63fff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb63fff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb63fff640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb63fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb63fff640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-45 : miss 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb63fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c0d61030:op=0x5608c12099d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c0d61030:op=0x5608c12099d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c0d60240:op=0x5608c11cc2a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c1207b70:op=0x5608c11cbcb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c11cd220:op=0x5608c12019f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c0d61030:op=0x5608c12099d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c0d60240:op=0x5608c11cc2a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c1207b70:op=0x5608c11cbcb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c11cd220:op=0x5608c12019f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c0d61030:op=0x5608c12099d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 15 stack 0x5608c0d61030 end 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 run: stack=0x5608c0d61030 is done 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c0d60240:op=0x5608c11cc2a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c1207b70:op=0x5608c11cbcb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c11cd220:op=0x5608c12019f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c0d5e8d0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c0d60240:op=0x5608c11cc2a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 15 stack 0x5608c0d60240 end 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 run: stack=0x5608c0d60240 is done 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c1207b70:op=0x5608c11cbcb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 15 stack 0x5608c1207b70 end 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 run: stack=0x5608c1207b70 is done 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c11cd220:op=0x5608c12019f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 15 stack 0x5608c11cd220 end 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 run: stack=0x5608c11cd220 is done 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c0d5e8d0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c0d5e8d0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c0d5e8d0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c0d5c440:op=0x5608c1209fc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c0d5c440:op=0x5608c1209fc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c0d5c440:op=0x5608c1209fc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c0d5c440:op=0x5608c1209fc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 15 stack 0x5608c0d5c440 end 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 run: stack=0x5608c0d5c440 is done 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c0d5e8d0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c0d603c0:op=0x5608c1201400:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c0d603c0:op=0x5608c1201400:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c0d603c0:op=0x5608c1201400:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c0d603c0:op=0x5608c1201400:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 15 stack 0x5608c0d603c0 end 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 run: stack=0x5608c0d603c0 is done 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c0d5e8d0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11c4f30:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11c4f30:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 enqueued request req=0x5608c0e57e00 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 req: 0x5608c0e57e00 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is io blocked 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb637fe640 20 dequeued request req=0x5608c0e57e00 2026-03-20T11:46:49.893 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb637fe640 20 RGWWQ: empty 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb68ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb68ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-49 info.flags=0x0 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb68ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-49 to cache LRU end 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb68ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9): adding source pipe: {s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb68ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9): adding dest pipe: {s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb68ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb68ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb68ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c0d6eed0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c0d6eed0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c0d6eed0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c0d6eed0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11cb550:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 run: stack=0x5608c11c0950 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d5e8d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d5e8d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb63fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb63fff640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-45 info.flags=0x0 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb63fff640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-45 to cache LRU end 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb63fff640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-45 : miss 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb63fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 6a65e8ef47a10d8de0ed2cf6fa7df64d8a7130387cf22baf5c88a50c544bd9c9 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114649Z 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:6a65e8ef47a10d8de0ed2cf6fa7df64d8a7130387cf22baf5c88a50c544bd9c9 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = b2e06f42e10f7093bb9deb57dcdd6fad28b7798f0febd2e3e8cb81d2e99bdf8e 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b2e06f42e10f7093bb9deb57dcdd6fad28b7798f0febd2e3e8cb81d2e99bdf8e 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114649Z 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:49.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=5, curl_handle=0x5608c0d60ac0 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdbacf23900 20 run: stack=0x5608c11d3560 is io blocked 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb067fc640 20 link_request req_data=0x5608c0d61210 req_data->id=5, curl_handle=0x5608c0d60ac0 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb63fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb63fff640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-45 info.flags=0x0 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb63fff640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-45 to cache LRU end 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb63fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5): adding source pipe: {s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb63fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5): adding dest pipe: {s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb63fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb63fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.889+0000 7fdb63fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c0d5f530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c0d5f530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c0d5f530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c0d5f530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c12042e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 20 run: stack=0x5608c0d61b20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1208d00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1208d00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 2c4c3d65d09d67d424095afb377b1bf45fb0571a4fef6f6f7adef71d5e64f543 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114649Z 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2c4c3d65d09d67d424095afb377b1bf45fb0571a4fef6f6f7adef71d5e64f543 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = f26ffd196f8827466c2ca3d1a16910cac2abaf4ac9a328a138c625a04d9550bc 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=f26ffd196f8827466c2ca3d1a16910cac2abaf4ac9a328a138c625a04d9550bc 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114649Z 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=6, curl_handle=0x5608c0d5bb20 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is io blocked 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.890+0000 7fdb067fc640 20 link_request req_data=0x5608c0d5c620 req_data->id=6, curl_handle=0x5608c0d5bb20 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.891+0000 7fdb637fe640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 info.flags=0x11 2026-03-20T11:46:49.895 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.891+0000 7fdb637fe640 10 rgw async rados processor: moving a1.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 to cache LRU end 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.891+0000 7fdb637fe640 10 rgw async rados processor: distributing notification oid=a1.rgw.control:notify.6 cni=[op: 0, obj: a1.rgw.meta:root:.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21, ofs0, ns] 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.891+0000 7fdb777fe640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 343597383949 cookie 94595599806640 notifier 6533 bl.length()=617 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.891+0000 7fdb777fe640 10 rgw watcher librados: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 info.flags=0x11 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.891+0000 7fdb777fe640 10 rgw watcher librados: moving a1.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 to cache LRU end 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdb637fe640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-61[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21]): orig_sources= new_sources= 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdb637fe640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-61[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21]): potential sources added=[] removed=[] 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdb637fe640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-61[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21]): orig_dests= new_dests= 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdb637fe640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-61[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21]): potential dests added=[] removed=[] 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11c4f30:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11c4f30:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11c4f30:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11c4f30:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdbacf23900 20 trim: trimmed bucket instance dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdbacf23900 15 stack 0x5608c10a18e0 end 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is done 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdbacf23900 4 starting trim on bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11d12f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11d12f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdbacf23900 20 enqueued request req=0x5608c11d6400 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdbacf23900 20 req: 0x5608c11d6400 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is io blocked 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdb62ffd640 20 dequeued request req=0x5608c11d6400 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdb62ffd640 20 RGWWQ: empty 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdb62ffd640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 : miss 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdb62ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdb62ffd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdb62ffd640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 info.flags=0x17 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdb62ffd640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 to cache LRU end 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdb62ffd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdb62ffd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdb62ffd640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdb62ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdb62ffd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-33 : miss 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdb62ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdb62ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdb62ffd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-33 info.flags=0x0 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdb62ffd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-33 to cache LRU end 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdb62ffd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-33 : miss 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.893+0000 7fdb62ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdb62ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdb62ffd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-33 info.flags=0x0 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdb62ffd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-33 to cache LRU end 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdb62ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38): adding source pipe: {s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdb62ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38): adding dest pipe: {s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdb62ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdb62ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdb62ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11d12f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11d12f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11d12f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11d12f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11d12f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.896 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c0d6eed0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c0d6eed0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = ca2640f8aec4219f5f260dbd7250c9d32ffb33f3e015592b8f75d46a5fdbebac 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114649Z 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:ca2640f8aec4219f5f260dbd7250c9d32ffb33f3e015592b8f75d46a5fdbebac 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = afc884bfbb6de51b067580cf3149def7303b84068101c0889369197414f0503f 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=afc884bfbb6de51b067580cf3149def7303b84068101c0889369197414f0503f 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114649Z 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=7, curl_handle=0x5608c11cd8e0 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is io blocked 2026-03-20T11:46:49.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.894+0000 7fdb067fc640 20 link_request req_data=0x7fdb400065e0 req_data->id=7, curl_handle=0x5608c11cd8e0 2026-03-20T11:46:49.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c0d614f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c0d614f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:49.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c0d5ef00:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdbacf23900 15 stack 0x5608c11c4c90 end 2026-03-20T11:46:49.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is done 2026-03-20T11:46:49.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdbacf23900 20 collect(): s=0x5608c11fb6f0 stack=0x5608c11c4c90 is complete 2026-03-20T11:46:49.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:49.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdbacf23900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:49.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdbacf23900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:49.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:49.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdbacf23900 15 stack 0x5608c11fb6f0 end 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdbacf23900 20 stack->operate() returned ret=-22 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdbacf23900 20 run: stack=0x5608c11fb6f0 is done 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdbacf23900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdbacf23900 4 starting trim on bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdbacf23900 20 enqueued request req=0x5608c11c6a20 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdbacf23900 20 req: 0x5608c11c6a20 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdbacf23900 20 run: stack=0x5608c11fb6f0 is io blocked 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdb627fc640 20 dequeued request req=0x5608c11c6a20 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdb627fc640 20 RGWWQ: empty 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdb627fc640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 : miss 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdb627fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdb627fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdb627fc640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 info.flags=0x17 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdb627fc640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 to cache LRU end 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdb627fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdb627fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdb627fc640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdb627fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdb627fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-47 : miss 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.924+0000 7fdb627fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdb627fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdb627fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-47 info.flags=0x0 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdb627fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-47 to cache LRU end 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdb627fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-47 : miss 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdb627fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdb627fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdb627fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-47 info.flags=0x0 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdb627fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-47 to cache LRU end 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdb627fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7): adding source pipe: {s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdb627fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7): adding dest pipe: {s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdb627fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdb627fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdb627fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:46:49.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c0d5ef00:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 20 run: stack=0x5608c11fb6f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c0d614f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c0d614f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = a9d378961f10bed529eecbcea9b4ab5230fdc7ccb5158b9604d14098aa71d2ac 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114649Z 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:a9d378961f10bed529eecbcea9b4ab5230fdc7ccb5158b9604d14098aa71d2ac 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = eaa3135709256a6b24dcf3e4acd2cddea149f0fdc0198a148c0b321785f7bee8 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=eaa3135709256a6b24dcf3e4acd2cddea149f0fdc0198a148c0b321785f7bee8 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114649Z 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=8, curl_handle=0x5608c11c9480 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is io blocked 2026-03-20T11:46:49.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.925+0000 7fdb067fc640 20 link_request req_data=0x5608c11c9b80 req_data->id=8, curl_handle=0x5608c11c9480 2026-03-20T11:46:49.923 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:49.925+0000 7fcfc39d5640 -1 req 431502630503857432 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d5e8d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d5e8d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d5e8d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d5e8d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11cb550:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdbacf23900 15 stack 0x5608c11d3560 end 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdbacf23900 20 run: stack=0x5608c11d3560 is done 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdbacf23900 20 collect(): s=0x5608c11c0950 stack=0x5608c11d3560 is complete 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdbacf23900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdbacf23900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdbacf23900 15 stack 0x5608c11c0950 end 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdbacf23900 20 stack->operate() returned ret=-22 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdbacf23900 20 run: stack=0x5608c11c0950 is done 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdbacf23900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdbacf23900 4 starting trim on bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdbacf23900 20 enqueued request req=0x5608c11d5030 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdbacf23900 20 req: 0x5608c11d5030 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdbacf23900 20 run: stack=0x5608c11c0950 is io blocked 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdb61ffb640 20 dequeued request req=0x5608c11d5030 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdb61ffb640 20 RGWWQ: empty 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdb61ffb640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 : miss 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdb61ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdb61ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdb61ffb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 info.flags=0x17 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdb61ffb640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 to cache LRU end 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdb61ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdb61ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdb61ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdb61ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-58 : miss 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.931+0000 7fdb61ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb61ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb61ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-58 info.flags=0x0 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb61ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-58 to cache LRU end 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb61ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-58 : miss 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb61ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1208d00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1208d00:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c12042e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 15 stack 0x5608c11d3250 end 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is done 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 collect(): s=0x5608c0d61b20 stack=0x5608c11d3250 is complete 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 15 stack 0x5608c0d61b20 end 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 stack->operate() returned ret=-22 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 run: stack=0x5608c0d61b20 is done 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 4 starting trim on bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c0d5f530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c0d5f530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 enqueued request req=0x5608c11bff40 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 req: 0x5608c11bff40 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 run: stack=0x5608c0d61b20 is io blocked 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb617fa640 20 dequeued request req=0x5608c11bff40 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb617fa640 20 RGWWQ: empty 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb617fa640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 : miss 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb617fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb61ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb61ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-58 info.flags=0x0 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb61ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-58 to cache LRU end 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb61ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18): adding source pipe: {s={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb61ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18): adding dest pipe: {s={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb61ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb61ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb61ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c12042e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 run: stack=0x5608c11c0950 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1208d00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1208d00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = b552708585732e7fb671c4ccdbcf7141906e44c8d5e2602d35830e4f3cc450dc 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114649Z 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:b552708585732e7fb671c4ccdbcf7141906e44c8d5e2602d35830e4f3cc450dc 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = a74a9870f0264d729667d69a72b72bdb8231e195e43a1113d350139e10b46eaa 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a74a9870f0264d729667d69a72b72bdb8231e195e43a1113d350139e10b46eaa 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114649Z 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=9, curl_handle=0x5608c0d5bb20 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is io blocked 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb067fc640 20 link_request req_data=0x5608c0d5c620 req_data->id=9, curl_handle=0x5608c0d5bb20 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb617fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb617fa640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 info.flags=0x17 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb617fa640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 to cache LRU end 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb617fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb617fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb617fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb617fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-30 : miss 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb617fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb617fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb617fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-30 info.flags=0x0 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb617fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-30 to cache LRU end 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb617fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-30 : miss 2026-03-20T11:46:49.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.932+0000 7fdb617fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdb617fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdb617fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-30 info.flags=0x0 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdb617fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-30 to cache LRU end 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdb617fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35): adding source pipe: {s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdb617fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35): adding dest pipe: {s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdb617fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdb617fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdb617fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c0d5f530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c0d5f530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c0d5f530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c0d5f530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11cb550:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 20 run: stack=0x5608c0d61b20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d5e8d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d5e8d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 86b11db8d510cdd3aa50a6753319b741d51e5bbd89a943411b17d4a528930b78 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114649Z 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:86b11db8d510cdd3aa50a6753319b741d51e5bbd89a943411b17d4a528930b78 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 8b6f1ce35bc755acc036e3957be67b10db6e6a1e181d41d5892e7dd5aa453315 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=8b6f1ce35bc755acc036e3957be67b10db6e6a1e181d41d5892e7dd5aa453315 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114649Z 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:49.931 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=10, curl_handle=0x5608c0d60ac0 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdbacf23900 20 run: stack=0x5608c11d3560 is io blocked 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.934+0000 7fdb067fc640 20 link_request req_data=0x5608c0d61210 req_data->id=10, curl_handle=0x5608c0d60ac0 2026-03-20T11:46:49.932 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:49.933+0000 7fd0b9bc1640 -1 req 86238580161711152 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c0d6eed0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c0d6eed0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11d12f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdbacf23900 15 stack 0x5608c11c5f10 end 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is done 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdbacf23900 20 collect(): s=0x5608c10a18e0 stack=0x5608c11c5f10 is complete 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdbacf23900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdbacf23900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdbacf23900 15 stack 0x5608c10a18e0 end 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdbacf23900 20 stack->operate() returned ret=-22 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is done 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdbacf23900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdbacf23900 4 starting trim on bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdbacf23900 20 enqueued request req=0x5608c11d6400 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdbacf23900 20 req: 0x5608c11d6400 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is io blocked 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdb60ff9640 20 dequeued request req=0x5608c11d6400 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdb60ff9640 20 RGWWQ: empty 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdb60ff9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 : miss 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdb60ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdb60ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdb60ff9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 info.flags=0x17 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdb60ff9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 to cache LRU end 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdb60ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdb60ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdb60ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdb60ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-20 : miss 2026-03-20T11:46:49.932 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdb60ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdb60ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdb60ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-20 info.flags=0x0 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdb60ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-20 to cache LRU end 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdb60ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-20 : miss 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.935+0000 7fdb60ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdb60ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdb60ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-20 info.flags=0x0 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdb60ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-20 to cache LRU end 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdb60ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25): adding source pipe: {s={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdb60ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25): adding dest pipe: {s={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdb60ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdb60ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdb60ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1208500:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11d5690:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11d5690:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 5d9237395ece7021b417efa4ca3dda8ad39683bf674c9dd7476e69bb1c27adaa 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114649Z 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:5d9237395ece7021b417efa4ca3dda8ad39683bf674c9dd7476e69bb1c27adaa 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 64d8546d1d8b202328dfe90309e5545af9b7c4b9e1c024bc3d32ebeee47d31ac 2026-03-20T11:46:49.933 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=64d8546d1d8b202328dfe90309e5545af9b7c4b9e1c024bc3d32ebeee47d31ac 2026-03-20T11:46:49.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114649Z 2026-03-20T11:46:49.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:49.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=11, curl_handle=0x5608c11cd8e0 2026-03-20T11:46:49.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is io blocked 2026-03-20T11:46:49.934 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.936+0000 7fdb067fc640 20 link_request req_data=0x7fdb400065e0 req_data->id=11, curl_handle=0x5608c11cd8e0 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c0d614f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c0d614f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c0d5ef00:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdbacf23900 15 stack 0x5608c11c4c90 end 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is done 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdbacf23900 20 collect(): s=0x5608c11fb6f0 stack=0x5608c11c4c90 is complete 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdbacf23900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdbacf23900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdbacf23900 15 stack 0x5608c11fb6f0 end 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdbacf23900 20 stack->operate() returned ret=-22 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdbacf23900 20 run: stack=0x5608c11fb6f0 is done 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdbacf23900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdbacf23900 4 starting trim on bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c0d6b640:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c0d6b640:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdbacf23900 20 enqueued request req=0x5608c11c6a20 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdbacf23900 20 req: 0x5608c11c6a20 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdbacf23900 20 run: stack=0x5608c11fb6f0 is io blocked 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdb607f8640 20 dequeued request req=0x5608c11c6a20 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdb607f8640 20 RGWWQ: empty 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdb607f8640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 : miss 2026-03-20T11:46:49.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.967+0000 7fdb607f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.968+0000 7fdb607f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:49.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.968+0000 7fdb607f8640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 info.flags=0x17 2026-03-20T11:46:49.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.968+0000 7fdb607f8640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 to cache LRU end 2026-03-20T11:46:49.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.968+0000 7fdb607f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:49.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.968+0000 7fdb607f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:49.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.968+0000 7fdb607f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:49.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.968+0000 7fdb607f8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-52 : miss 2026-03-20T11:46:49.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.968+0000 7fdb607f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.968+0000 7fdb607f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.968+0000 7fdb607f8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-52 info.flags=0x0 2026-03-20T11:46:49.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.968+0000 7fdb607f8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-52 to cache LRU end 2026-03-20T11:46:49.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.968+0000 7fdb607f8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-52 : miss 2026-03-20T11:46:49.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.968+0000 7fdb607f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdb607f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdb607f8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-52 info.flags=0x0 2026-03-20T11:46:49.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdb607f8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-52 to cache LRU end 2026-03-20T11:46:49.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdb607f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12): adding source pipe: {s={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdb607f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12): adding dest pipe: {s={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdb607f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdb607f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdb607f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c0d6b640:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c0d6b640:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c0d6b640:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c0d6b640:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c0d5ef00:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 20 run: stack=0x5608c11fb6f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c0d614f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c0d614f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 6301ec6784d3c24e0f6100e25f636700ab1c91335ca3da0e1680d76c0dbe96f0 2026-03-20T11:46:49.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:49.967 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114649Z 2026-03-20T11:46:49.967 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:49.967 INFO:teuthology.orchestra.run.vm02.stderr:6301ec6784d3c24e0f6100e25f636700ab1c91335ca3da0e1680d76c0dbe96f0 2026-03-20T11:46:49.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:49.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:49.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:49.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:49.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = debdecbb7c0808f81afe85890ac8989801cf4acb7e3c2a8e30c5012b292b0e0b 2026-03-20T11:46:49.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=debdecbb7c0808f81afe85890ac8989801cf4acb7e3c2a8e30c5012b292b0e0b 2026-03-20T11:46:49.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114649Z 2026-03-20T11:46:49.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:49.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=12, curl_handle=0x5608c11c9480 2026-03-20T11:46:49.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is io blocked 2026-03-20T11:46:49.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.969+0000 7fdb067fc640 20 link_request req_data=0x5608c11c9b80 req_data->id=12, curl_handle=0x5608c11c9480 2026-03-20T11:46:49.972 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1208d00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.972 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1208d00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.972 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d5e8d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.972 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d5e8d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:49.972 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1208d00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.972 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11cb550:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.972 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 15 stack 0x5608c11d3560 end 2026-03-20T11:46:49.972 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 run: stack=0x5608c11d3560 is done 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1208d00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 collect(): s=0x5608c0d61b20 stack=0x5608c11d3560 is complete 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 15 stack 0x5608c0d61b20 end 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 stack->operate() returned ret=-22 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 run: stack=0x5608c0d61b20 is done 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c12042e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 15 stack 0x5608c11d3250 end 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is done 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 collect(): s=0x5608c11c0950 stack=0x5608c11d3250 is complete 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 10 trimming bilogs for bucket=:dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) markers=[,00000000001.77.6,,,,00000000001.65.6,,,00000000001.20.6,,00000000001.28.6], shards=11 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 4 starting trim on bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c4f30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 10 trimming bilog shard 1 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.77.6 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 10 trimming bilog shard 5 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.65.6 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 10 trimming bilog shard 8 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.20.6 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 10 trimming bilog shard 10 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.28.6 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 run: stack=0x5608c11c0950 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c0d6cbe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.973 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c0d5f530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c12099d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c11d1130:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11d1df0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c0d6cbe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 enqueued request req=0x5608c1202eb0 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 req: 0x5608c1202eb0 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 run: stack=0x5608c0d61b20 is io blocked 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c0d5f530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.1 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdb5fff7640 20 dequeued request req=0x5608c1202eb0 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdb5fff7640 20 RGWWQ: empty 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is io blocked 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c12099d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.5 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 run: stack=0x5608c11d3560 is io blocked 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c11d1130:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.8 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is io blocked 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11d1df0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.10 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdb5fff7640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 : miss 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is io blocked 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.975+0000 7fdb5fff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.976+0000 7fdb5fff7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.976+0000 7fdb5fff7640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 info.flags=0x17 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.976+0000 7fdb5fff7640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 to cache LRU end 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.976+0000 7fdb5fff7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.976+0000 7fdb5fff7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.976+0000 7fdb5fff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdb5fff7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-42 : miss 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdb5fff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c0d5f530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c0d5f530:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 15 stack 0x5608c11d3250 end 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is done 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c4f30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 20 run: stack=0x5608c11c0950 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c11d1130:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c11d1130:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 15 stack 0x5608c0d6dec0 end 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is done 2026-03-20T11:46:49.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c4f30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 20 run: stack=0x5608c11c0950 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdb5fff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdb5fff7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-42 info.flags=0x0 2026-03-20T11:46:49.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdb5fff7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-42 to cache LRU end 2026-03-20T11:46:49.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdb5fff7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-42 : miss 2026-03-20T11:46:49.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdb5fff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11d1df0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11d1df0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 15 stack 0x5608c11c5dd0 end 2026-03-20T11:46:49.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is done 2026-03-20T11:46:49.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c4f30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 20 run: stack=0x5608c11c0950 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c12099d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c12099d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 15 stack 0x5608c11d3560 end 2026-03-20T11:46:49.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 20 run: stack=0x5608c11d3560 is done 2026-03-20T11:46:49.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c4f30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 20 trim: trimmed bucket instance dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:49.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.977+0000 7fdbacf23900 15 stack 0x5608c11c0950 end 2026-03-20T11:46:49.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 run: stack=0x5608c11c0950 is done 2026-03-20T11:46:49.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:49.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 4 starting trim on bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c0d5f530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c0d5f530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 enqueued request req=0x5608c0d5c050 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 req: 0x5608c0d5c050 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 run: stack=0x5608c11c0950 is io blocked 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdb5f7f6640 20 dequeued request req=0x5608c0d5c050 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdb5f7f6640 20 RGWWQ: empty 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdb5f7f6640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 : miss 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdb5f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11d5690:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdb5fff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdb5fff7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-42 info.flags=0x0 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdb5fff7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-42 to cache LRU end 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdb5fff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2): adding source pipe: {s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdb5fff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2): adding dest pipe: {s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdb5fff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdb5fff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdb5fff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11d5690:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c0d6cbe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11d5690:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c0d6cbe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11d5690:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c0d6cbe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1208500:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 15 stack 0x5608c11c5f10 end 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is done 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c0d6cbe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 collect(): s=0x5608c10a18e0 stack=0x5608c11c5f10 is complete 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 10 trimming bilogs for bucket=:dipepu-20[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25]) markers=[,,,,,,,,00000000005.42.14,,], shards=11 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11c4f30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 10 trimming bilog shard 8 of :dipepu-20[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25]) at marker 00000000005.42.14 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11cb550:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 run: stack=0x5608c0d61b20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11cc360:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c0d5e8d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.976 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11cc360:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25.8 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 run: stack=0x5608c11d3560 is io blocked 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c0d5e8d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdb5f7f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdb5f7f6640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 info.flags=0x17 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdb5f7f6640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 to cache LRU end 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdb5f7f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdb5f7f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdb5f7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdb5f7f6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-2 : miss 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdb5f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = d6f6d74d07f19ffe855c18b3d0324f6a6605b572bbeb93c3bf8ae2e46e391b8f 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114649Z 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:d6f6d74d07f19ffe855c18b3d0324f6a6605b572bbeb93c3bf8ae2e46e391b8f 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = cd7e661f0de2f68df602ded5b0528e4b6b1040779e10588a88d6a028135e8820 2026-03-20T11:46:49.977 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=cd7e661f0de2f68df602ded5b0528e4b6b1040779e10588a88d6a028135e8820 2026-03-20T11:46:49.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114649Z 2026-03-20T11:46:49.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:49.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=13, curl_handle=0x5608c11cd8e0 2026-03-20T11:46:49.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is io blocked 2026-03-20T11:46:49.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.978+0000 7fdb067fc640 20 link_request req_data=0x7fdb400065e0 req_data->id=13, curl_handle=0x5608c11cd8e0 2026-03-20T11:46:49.978 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:49.978+0000 7fcff1230640 -1 req 5914886177870620723 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:49.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.979+0000 7fdb5f7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.979+0000 7fdb5f7f6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-2 info.flags=0x0 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.979+0000 7fdb5f7f6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-2 to cache LRU end 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.979+0000 7fdb5f7f6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-2 : miss 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.979+0000 7fdb5f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.979+0000 7fdb5f7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.979+0000 7fdb5f7f6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-2 info.flags=0x0 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.979+0000 7fdb5f7f6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-2 to cache LRU end 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.979+0000 7fdb5f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2): adding source pipe: {s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.979+0000 7fdb5f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2): adding dest pipe: {s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11cc360:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11cc360:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 15 stack 0x5608c11d3560 end 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 run: stack=0x5608c11d3560 is done 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11c4f30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 trim: trimmed bucket instance dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 15 stack 0x5608c10a18e0 end 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is done 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 4 starting trim on bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11cc360:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11cc360:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 enqueued request req=0x5608c1201fe0 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 req: 0x5608c1201fe0 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is io blocked 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdb5f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdb5f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdb5f7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdb5f7f6640 20 dequeued request req=0x5608c1201fe0 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdb5f7f6640 20 RGWWQ: empty 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdb5f7f6640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 : miss 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c0d5f530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c0d5f530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c0d5f530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdb5f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c0d5f530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c12042e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 run: stack=0x5608c11c0950 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11d3a50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11d3a50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.979 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 700a3a59c60898e946188113909505de2daf63e68c1bb06b671e17e7178614a3 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114649Z 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:700a3a59c60898e946188113909505de2daf63e68c1bb06b671e17e7178614a3 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = fd711e4f02ab6bd6af7a019fd278e8cb154b231a17066f48fe553adf3d172ece 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=fd711e4f02ab6bd6af7a019fd278e8cb154b231a17066f48fe553adf3d172ece 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114649Z 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=14, curl_handle=0x5608c0d60ac0 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdbacf23900 20 run: stack=0x5608c11d3560 is io blocked 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdb067fc640 20 link_request req_data=0x5608c0d61210 req_data->id=14, curl_handle=0x5608c0d60ac0 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdb5f7f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdb5f7f6640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 info.flags=0x17 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdb5f7f6640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 to cache LRU end 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdb5f7f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdb5f7f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdb5f7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdb5f7f6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-57 : miss 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.980+0000 7fdb5f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdb5f7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdb5f7f6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-57 info.flags=0x0 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdb5f7f6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-57 to cache LRU end 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdb5f7f6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-57 : miss 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdb5f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdb5f7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdb5f7f6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-57 info.flags=0x0 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdb5f7f6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-57 to cache LRU end 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdb5f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17): adding source pipe: {s={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.980 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdb5f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17): adding dest pipe: {s={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdb5f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdb5f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdb5f7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11cc360:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11cc360:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11cc360:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11cc360:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c0d6e300:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11cc360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11cc360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:49 +0000 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114649Z 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 299fb4c918e759324358a8a7d5502445985363b81b13051223b9a0a8fa902fd1 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114649Z 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:299fb4c918e759324358a8a7d5502445985363b81b13051223b9a0a8fa902fd1 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 4840042156bd49af66ddcdb16808a0ce4a2ecc685422c3056271cfb0a9faee85 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=4840042156bd49af66ddcdb16808a0ce4a2ecc685422c3056271cfb0a9faee85 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114649Z 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=15, curl_handle=0x5608c0d5bb20 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is io blocked 2026-03-20T11:46:49.981 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:49.981+0000 7fdb067fc640 20 link_request req_data=0x5608c0d5c620 req_data->id=15, curl_handle=0x5608c0d5bb20 2026-03-20T11:46:49.981 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:49.980+0000 7fcfdca07640 -1 req 9275251071479118053 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:50.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c0d614f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c0d614f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c0d614f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c0d614f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c0d5ef00:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 15 stack 0x5608c11c4c90 end 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is done 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 collect(): s=0x5608c11fb6f0 stack=0x5608c11c4c90 is complete 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 10 trimming bilogs for bucket=:dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) markers=[00000000004.19.6,00000000002.34.1,00000000004.47.6,00000000002.22.1,00000000004.31.6,00000000002.67.1,00000000002.32.1,00000000004.16.6,00000000002.26.1,00000000002.76.1,00000000002.54.1], shards=11 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c4f30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 10 trimming bilog shard 0 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000004.19.6 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 10 trimming bilog shard 1 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.34.1 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 10 trimming bilog shard 2 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000004.47.6 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 10 trimming bilog shard 3 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.22.1 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 10 trimming bilog shard 4 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000004.31.6 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 10 trimming bilog shard 5 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.67.1 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 10 trimming bilog shard 6 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.32.1 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 10 trimming bilog shard 7 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000004.16.6 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 10 trimming bilog shard 8 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.26.1 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 10 trimming bilog shard 9 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.76.1 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 10 trimming bilog shard 10 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.54.1 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 run: stack=0x5608c11fb6f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11d5690:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1233f20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c12063f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c11cbb70:op=0x5608c12082f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c11d1130:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c1235dc0:op=0x5608c0d6b640:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c0d6cbe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c120a7e0:op=0x5608c1206cf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1205650:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c1232de0:op=0x5608c11d1d00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c11d2e70:op=0x5608c1233440:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11d5690:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.0 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is io blocked 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1233f20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.1 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is io blocked 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c12063f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.2 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is io blocked 2026-03-20T11:46:50.010 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c11cbb70:op=0x5608c12082f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.3 2026-03-20T11:46:50.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 run: stack=0x5608c11cbb70 is io blocked 2026-03-20T11:46:50.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c11d1130:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.4 2026-03-20T11:46:50.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 run: stack=0x5608c11c55c0 is io blocked 2026-03-20T11:46:50.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c1235dc0:op=0x5608c0d6b640:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.5 2026-03-20T11:46:50.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 run: stack=0x5608c1235dc0 is io blocked 2026-03-20T11:46:50.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c0d6cbe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.6 2026-03-20T11:46:50.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 run: stack=0x5608c0d5b650 is io blocked 2026-03-20T11:46:50.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c120a7e0:op=0x5608c1206cf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.7 2026-03-20T11:46:50.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 run: stack=0x5608c120a7e0 is io blocked 2026-03-20T11:46:50.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1205650:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.8 2026-03-20T11:46:50.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 run: stack=0x5608c11d6650 is io blocked 2026-03-20T11:46:50.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c1232de0:op=0x5608c11d1d00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.9 2026-03-20T11:46:50.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 run: stack=0x5608c1232de0 is io blocked 2026-03-20T11:46:50.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 cr:s=0x5608c11d2e70:op=0x5608c1233440:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.012+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.10 2026-03-20T11:46:50.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.013+0000 7fdbacf23900 20 run: stack=0x5608c11d2e70 is io blocked 2026-03-20T11:46:50.011 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.014+0000 7fdb55fe3640 20 rgw rados thread: process(): notifying mdlog change, shard_id=2 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.014+0000 7fdb55fe3640 20 rgw rados thread: cr:s=0x7fdb84075e70:op=0x7fdb84075fb0:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.014+0000 7fdb55fe3640 20 rgw rados thread: cr:s=0x7fdb84075e70:op=0x7fdb84075fb0:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.014+0000 7fdb55fe3640 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.014+0000 7fdb55fe3640 20 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.014+0000 7fdb55fe3640 20 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdb55fe3640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:038966de9f6b9a901b20b4c6ca8b2a46009feebe031babc842d43690c0bc222b 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdb55fe3640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]payload request hash = 038966de9f6b9a901b20b4c6ca8b2a46009feebe031babc842d43690c0bc222b 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdb55fe3640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]canonical request = POST 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:notify=&rgwx-zonegroup=a&type=metadata 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:038966de9f6b9a901b20b4c6ca8b2a46009feebe031babc842d43690c0bc222b 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:038966de9f6b9a901b20b4c6ca8b2a46009feebe031babc842d43690c0bc222b 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdb55fe3640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]canonical request hash = 66482a9a1b3017e6a0384f65dff54b9253295b078b7ade2609f6eff1c21b1a63 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdb55fe3640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:66482a9a1b3017e6a0384f65dff54b9253295b078b7ade2609f6eff1c21b1a63 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdb55fe3640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdb55fe3640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdb55fe3640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdb55fe3640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdb55fe3640 10 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]generated signature = 75a89ef83079859452654fce9dbfbe1de6043692a9d3830d3753408b9ef8b601 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdb55fe3640 20 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=75a89ef83079859452654fce9dbfbe1de6043692a9d3830d3753408b9ef8b601 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdb55fe3640 20 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: 038966de9f6b9a901b20b4c6ca8b2a46009feebe031babc842d43690c0bc222b 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdb55fe3640 20 http_client[POST/http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdb55fe3640 20 sending request to http://vm05.local:8001/admin/log?type=metadata¬ify&rgwx-zonegroup=a 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdb55fe3640 20 register_request mgr=0x5608c0d4f9a8 req_data->id=0, curl_handle=0x5608c11c9480 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdb55fe3640 20 run: stack=0x7fdb84075e70 is io blocked 2026-03-20T11:46:50.012 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1205650:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdb567e4640 20 link_request req_data=0x7fdb84079d90 req_data->id=0, curl_handle=0x5608c11c9480 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1205650:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 15 stack 0x5608c11d6650 end 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 run: stack=0x5608c11d6650 is done 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c4f30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 run: stack=0x5608c11fb6f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c0d6cbe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c0d6cbe0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 15 stack 0x5608c0d5b650 end 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 run: stack=0x5608c0d5b650 is done 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c11d1130:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c11d1130:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 15 stack 0x5608c11c55c0 end 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 run: stack=0x5608c11c55c0 is done 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 cr:s=0x5608c11cbb70:op=0x5608c12082f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 cr:s=0x5608c11cbb70:op=0x5608c12082f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 15 stack 0x5608c11cbb70 end 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 run: stack=0x5608c11cbb70 is done 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c12063f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c12063f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 15 stack 0x5608c11d3250 end 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is done 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c4f30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c4f30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c4f30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c4f30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 run: stack=0x5608c11fb6f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1233f20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1233f20:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.013 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 15 stack 0x5608c0d6dec0 end 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is done 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c4f30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 run: stack=0x5608c11fb6f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11d5690:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11d5690:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 15 stack 0x5608c11c4c90 end 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is done 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c4f30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.015+0000 7fdbacf23900 20 run: stack=0x5608c11fb6f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 cr:s=0x5608c120a7e0:op=0x5608c1206cf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 cr:s=0x5608c120a7e0:op=0x5608c1206cf0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 15 stack 0x5608c120a7e0 end 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 run: stack=0x5608c120a7e0 is done 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c4f30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 run: stack=0x5608c11fb6f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 cr:s=0x5608c11d2e70:op=0x5608c1233440:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 cr:s=0x5608c11d2e70:op=0x5608c1233440:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 15 stack 0x5608c11d2e70 end 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 run: stack=0x5608c11d2e70 is done 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c4f30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 run: stack=0x5608c11fb6f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 cr:s=0x5608c1235dc0:op=0x5608c0d6b640:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 cr:s=0x5608c1235dc0:op=0x5608c0d6b640:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 15 stack 0x5608c1235dc0 end 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 run: stack=0x5608c1235dc0 is done 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c4f30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 run: stack=0x5608c11fb6f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 cr:s=0x5608c1232de0:op=0x5608c11d1d00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 cr:s=0x5608c1232de0:op=0x5608c11d1d00:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 15 stack 0x5608c1232de0 end 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 run: stack=0x5608c1232de0 is done 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c4f30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 cr:s=0x5608c11fb6f0:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 trim: trimmed bucket instance dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 15 stack 0x5608c11fb6f0 end 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 run: stack=0x5608c11fb6f0 is done 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 4 starting trim on bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c12063f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c12063f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 enqueued request req=0x5608c11c67c0 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 req: 0x5608c11c67c0 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is io blocked 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdb5e7f4640 20 dequeued request req=0x5608c11c67c0 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdb5e7f4640 20 RGWWQ: empty 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdb5e7f4640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 : miss 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdb5e7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdb5e7f4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:50.014 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdb5e7f4640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 info.flags=0x17 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdb5e7f4640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 to cache LRU end 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdb5e7f4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdb5e7f4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdb5e7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdb5e7f4640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-29 : miss 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdb5e7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdb55fe3640 20 rgw rados thread: cr:s=0x7fdb84075e70:op=0x7fdb84075fb0:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdb55fe3640 20 rgw rados thread: cr:s=0x7fdb84075e70:op=0x7fdb84075fb0:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdb55fe3640 20 rgw rados thread: cr:s=0x7fdb84075e70:op=0x7fdb84075fb0:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdb55fe3640 20 rgw rados thread: cr:s=0x7fdb84075e70:op=0x7fdb84075fb0:21RGWPostRESTResourceCRISt3setIiSt4lessIiESaIiEEiiE: operate() 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdb55fe3640 15 stack 0x7fdb84075e70 end 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.016+0000 7fdb55fe3640 20 run: stack=0x7fdb84075e70 is done 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdb5e7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdb5e7f4640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-29 info.flags=0x0 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdb5e7f4640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-29 to cache LRU end 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdb5e7f4640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-29 : miss 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdb5e7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.015 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:50.017+0000 7fcfefa2d640 -1 req 18021393325476528774 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdb5e7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdb5e7f4640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-29 info.flags=0x0 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdb5e7f4640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-29 to cache LRU end 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdb5e7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34): adding source pipe: {s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdb5e7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34): adding dest pipe: {s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdb5e7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdb5e7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdb5e7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c12063f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c12063f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c12063f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c12063f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c0d614d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1236030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1236030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.015 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 1e51578cfc7de3a16cfbb39e4920b38fdd89808c417ff3d502a9b11e124e137c 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr:1e51578cfc7de3a16cfbb39e4920b38fdd89808c417ff3d502a9b11e124e137c 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 9bdd09c478be7b56ca212dc0a63fff5056a403d8a48e5f9606011d9c3f31dc85 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=9bdd09c478be7b56ca212dc0a63fff5056a403d8a48e5f9606011d9c3f31dc85 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=16, curl_handle=0x5608c11c9480 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is io blocked 2026-03-20T11:46:50.016 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.017+0000 7fdb067fc640 20 link_request req_data=0x5608c11cdff0 req_data->id=16, curl_handle=0x5608c11c9480 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c0d5e8d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c0d5e8d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11cb550:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdbacf23900 15 stack 0x5608c11c5f10 end 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is done 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdbacf23900 20 collect(): s=0x5608c0d61b20 stack=0x5608c11c5f10 is complete 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdbacf23900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdbacf23900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdbacf23900 15 stack 0x5608c0d61b20 end 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdbacf23900 20 stack->operate() returned ret=-22 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdbacf23900 20 run: stack=0x5608c0d61b20 is done 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdbacf23900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdbacf23900 4 starting trim on bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdbacf23900 20 enqueued request req=0x5608c11d6400 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdbacf23900 20 req: 0x5608c11d6400 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdbacf23900 20 run: stack=0x5608c0d61b20 is io blocked 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdb5dff3640 20 dequeued request req=0x5608c11d6400 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdb5dff3640 20 RGWWQ: empty 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdb5dff3640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 : miss 2026-03-20T11:46:50.018 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdb5dff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdb5dff3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdb5dff3640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 info.flags=0x17 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdb5dff3640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 to cache LRU end 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdb5dff3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdb5dff3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdb5dff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdb5dff3640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-23 : miss 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.021+0000 7fdb5dff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11d3a50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11d3a50:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c12042e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 15 stack 0x5608c11d3560 end 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 run: stack=0x5608c11d3560 is done 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 collect(): s=0x5608c11c0950 stack=0x5608c11d3560 is complete 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 cr:s=0x5608c11c0950:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 15 stack 0x5608c11c0950 end 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 stack->operate() returned ret=-22 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 run: stack=0x5608c11c0950 is done 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 4 starting trim on bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d6b640:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d6b640:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 enqueued request req=0x5608c0d5c050 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 req: 0x5608c0d5c050 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 run: stack=0x5608c11d3560 is io blocked 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5d7f2640 20 dequeued request req=0x5608c0d5c050 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5d7f2640 20 RGWWQ: empty 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5d7f2640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 : miss 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5d7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5dff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5dff3640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-23 info.flags=0x0 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5dff3640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-23 to cache LRU end 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5dff3640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-23 : miss 2026-03-20T11:46:50.019 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5dff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5d7f2640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5d7f2640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 info.flags=0x17 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5d7f2640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 to cache LRU end 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5d7f2640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5d7f2640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5d7f2640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5d7f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5d7f2640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-34 : miss 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5d7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5dff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5dff3640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-23 info.flags=0x0 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5dff3640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-23 to cache LRU end 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28): adding source pipe: {s={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28): adding dest pipe: {s={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5d7f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5d7f2640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-34 info.flags=0x0 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5d7f2640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-34 to cache LRU end 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5d7f2640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-34 : miss 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5dff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdb5d7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c12042e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 run: stack=0x5608c0d61b20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1201240:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1201240:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 19cba08da9453342256de8e4f674ba9c8970b69d2ceff8eb78206f5ad1fc0b6e 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:19cba08da9453342256de8e4f674ba9c8970b69d2ceff8eb78206f5ad1fc0b6e 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = a3a8d309f703d90723862007dc0a879abdb780836eddcb8adf452bd591395911 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.022+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a3a8d309f703d90723862007dc0a879abdb780836eddcb8adf452bd591395911 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=17, curl_handle=0x5608c0d60ac0 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is io blocked 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdb067fc640 20 link_request req_data=0x5608c11c6dc0 req_data->id=17, curl_handle=0x5608c0d60ac0 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdb5d7f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdb5d7f2640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-34 info.flags=0x0 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdb5d7f2640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-34 to cache LRU end 2026-03-20T11:46:50.021 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdb5d7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39): adding source pipe: {s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdb5d7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39): adding dest pipe: {s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdb5d7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdb5d7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdb5d7f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d6b640:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d6b640:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d6b640:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d6b640:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c0d6b640:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 20 run: stack=0x5608c11d3560 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c11d5690:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c11d5690:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = ad9e26410ea4aa25c9a288d61e4c078f81c2de3a8857bca4b3b3e2bb3dfc87cd 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:ad9e26410ea4aa25c9a288d61e4c078f81c2de3a8857bca4b3b3e2bb3dfc87cd 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.022 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 36bf46c88a84875a340aea9a8043554bd37c148960c3b773558edb907983af03 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=36bf46c88a84875a340aea9a8043554bd37c148960c3b773558edb907983af03 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=18, curl_handle=0x5608c11cd8e0 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdbacf23900 20 run: stack=0x5608c0d5b650 is io blocked 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.023+0000 7fdb067fc640 20 link_request req_data=0x5608c11c52f0 req_data->id=18, curl_handle=0x5608c11cd8e0 2026-03-20T11:46:50.023 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:50.022+0000 7fcfd9200640 -1 req 18308990728688275299 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.024+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11cc360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.024+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11cc360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.024+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11cc360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.024+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11cc360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.024+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c0d6e300:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.024+0000 7fdbacf23900 15 stack 0x5608c11c5dd0 end 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.024+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is done 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.024+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.024+0000 7fdbacf23900 20 collect(): s=0x5608c10a18e0 stack=0x5608c11c5dd0 is complete 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.024+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.024+0000 7fdbacf23900 10 trimming bilogs for bucket=:dipepu-57[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17]) markers=[,00000000001.41.6,,,,00000000001.23.6,,,,,], shards=11 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.024+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c1237a60:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.024+0000 7fdbacf23900 10 trimming bilog shard 1 of :dipepu-57[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17]) at marker 00000000001.41.6 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.024+0000 7fdbacf23900 10 trimming bilog shard 5 of :dipepu-57[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17]) at marker 00000000001.23.6 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.024+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.024+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11d46b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.024+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c1233f20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.024+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11d46b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.024+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17.1 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.024+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is io blocked 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.024+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c1233f20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.024+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17.5 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.024+0000 7fdbacf23900 20 run: stack=0x5608c11c55c0 is io blocked 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c1233f20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c1233f20:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdbacf23900 15 stack 0x5608c11c55c0 end 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdbacf23900 20 run: stack=0x5608c11c55c0 is done 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c1237a60:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11d46b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11d46b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdbacf23900 15 stack 0x5608c11c5dd0 end 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is done 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c1237a60:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdbacf23900 20 trim: trimmed bucket instance dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdbacf23900 15 stack 0x5608c10a18e0 end 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is done 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdbacf23900 4 starting trim on bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123e790:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123e790:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdbacf23900 20 enqueued request req=0x5608c1201fe0 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdbacf23900 20 req: 0x5608c1201fe0 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is io blocked 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdb5cff1640 20 dequeued request req=0x5608c1201fe0 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdb5cff1640 20 RGWWQ: empty 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdb5cff1640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 : miss 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdb5cff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdb5cff1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdb5cff1640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 info.flags=0x17 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdb5cff1640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 to cache LRU end 2026-03-20T11:46:50.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdb5cff1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdb5cff1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.026+0000 7fdb5cff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdb5cff1640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-54 : miss 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdb5cff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdb5cff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdb5cff1640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-54 info.flags=0x0 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdb5cff1640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-54 to cache LRU end 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdb5cff1640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-54 : miss 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdb5cff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdb5cff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdb5cff1640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-54 info.flags=0x0 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdb5cff1640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-54 to cache LRU end 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdb5cff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14): adding source pipe: {s={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdb5cff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14): adding dest pipe: {s={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdb5cff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdb5cff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdb5cff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123e790:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123e790:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123e790:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123e790:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c0d6e300:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11cc360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11cc360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.025 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.026 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.026 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = b2ce035491ab4e4be6003c495fd269ab1ece5e45913899127d0814d20015e69e 2026-03-20T11:46:50.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.026 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.026 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.026 INFO:teuthology.orchestra.run.vm02.stderr:b2ce035491ab4e4be6003c495fd269ab1ece5e45913899127d0814d20015e69e 2026-03-20T11:46:50.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 52e0c1c466861d1780860d568a8ca23733112ceb725687957b5d3cc3e1f011c0 2026-03-20T11:46:50.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=52e0c1c466861d1780860d568a8ca23733112ceb725687957b5d3cc3e1f011c0 2026-03-20T11:46:50.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=19, curl_handle=0x5608c0d5bb20 2026-03-20T11:46:50.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is io blocked 2026-03-20T11:46:50.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.027+0000 7fdb067fc640 20 link_request req_data=0x5608c12688c0 req_data->id=19, curl_handle=0x5608c0d5bb20 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1236030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1236030:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c0d614d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdbacf23900 15 stack 0x5608c0d6dec0 end 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is done 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdbacf23900 20 collect(): s=0x5608c11c4c90 stack=0x5608c0d6dec0 is complete 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdbacf23900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdbacf23900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdbacf23900 15 stack 0x5608c11c4c90 end 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdbacf23900 20 stack->operate() returned ret=-22 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is done 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdbacf23900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdbacf23900 4 starting trim on bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c12082f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c12082f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdbacf23900 20 enqueued request req=0x5608c11c67c0 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdbacf23900 20 req: 0x5608c11c67c0 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is io blocked 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdb5c7f0640 20 dequeued request req=0x5608c11c67c0 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdb5c7f0640 20 RGWWQ: empty 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdb5c7f0640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 : miss 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdb5c7f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdb5c7f0640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdb5c7f0640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 info.flags=0x17 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdb5c7f0640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 to cache LRU end 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdb5c7f0640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdb5c7f0640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdb5c7f0640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdb5c7f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdb5c7f0640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-32 : miss 2026-03-20T11:46:50.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.059+0000 7fdb5c7f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdb5c7f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdb5c7f0640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-32 info.flags=0x0 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdb5c7f0640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-32 to cache LRU end 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdb5c7f0640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-32 : miss 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdb5c7f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdb5c7f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdb5c7f0640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-32 info.flags=0x0 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdb5c7f0640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-32 to cache LRU end 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdb5c7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37): adding source pipe: {s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdb5c7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37): adding dest pipe: {s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdb5c7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdb5c7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdb5c7f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c12082f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c12082f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c12082f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c12082f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c0d614d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c12082f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c12082f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.057 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.058 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.058 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.058 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 4d6f6a7952e7211ce1bb902248eec70f6d7d6654ff1eaf4f752e1935211ee220 2026-03-20T11:46:50.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.058 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.058 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.058 INFO:teuthology.orchestra.run.vm02.stderr:4d6f6a7952e7211ce1bb902248eec70f6d7d6654ff1eaf4f752e1935211ee220 2026-03-20T11:46:50.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 139b105e44eb0eec5aadbaa6c968172c17215af740666beaea2d9ea9ee6e9403 2026-03-20T11:46:50.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=139b105e44eb0eec5aadbaa6c968172c17215af740666beaea2d9ea9ee6e9403 2026-03-20T11:46:50.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=20, curl_handle=0x5608c11c9480 2026-03-20T11:46:50.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is io blocked 2026-03-20T11:46:50.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.060+0000 7fdb067fc640 20 link_request req_data=0x5608c1208920 req_data->id=20, curl_handle=0x5608c11c9480 2026-03-20T11:46:50.058 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:50.060+0000 7fd06b324640 -1 req 13945308845188244115 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c11d5690:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c11d5690:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c0d6b640:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdbacf23900 15 stack 0x5608c0d5b650 end 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdbacf23900 20 run: stack=0x5608c0d5b650 is done 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdbacf23900 20 collect(): s=0x5608c11d3560 stack=0x5608c0d5b650 is complete 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdbacf23900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdbacf23900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdbacf23900 15 stack 0x5608c11d3560 end 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdbacf23900 20 stack->operate() returned ret=-22 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdbacf23900 20 run: stack=0x5608c11d3560 is done 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdbacf23900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdbacf23900 4 starting trim on bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d6b640:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d6b640:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdbacf23900 20 enqueued request req=0x5608c0d5c050 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdbacf23900 20 req: 0x5608c0d5c050 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdbacf23900 20 run: stack=0x5608c11d3560 is io blocked 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdb5bfef640 20 dequeued request req=0x5608c0d5c050 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdb5bfef640 20 RGWWQ: empty 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdb5bfef640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 : miss 2026-03-20T11:46:50.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdb5bfef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdb5bfef640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:50.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdb5bfef640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 info.flags=0x17 2026-03-20T11:46:50.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdb5bfef640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 to cache LRU end 2026-03-20T11:46:50.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdb5bfef640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdb5bfef640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdb5bfef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:46:50.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdb5bfef640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-19 : miss 2026-03-20T11:46:50.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.064+0000 7fdb5bfef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdb5bfef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdb5bfef640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-19 info.flags=0x0 2026-03-20T11:46:50.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdb5bfef640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-19 to cache LRU end 2026-03-20T11:46:50.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdb5bfef640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-19 : miss 2026-03-20T11:46:50.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdb5bfef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdb5bfef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdb5bfef640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-19 info.flags=0x0 2026-03-20T11:46:50.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdb5bfef640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-19 to cache LRU end 2026-03-20T11:46:50.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdb5bfef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24): adding source pipe: {s={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdb5bfef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24): adding dest pipe: {s={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdb5bfef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdb5bfef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdb5bfef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:46:50.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d6b640:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d6b640:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d6b640:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d6b640:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c11d5690:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 20 run: stack=0x5608c11d3560 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c0d6b640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c0d6b640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 937cdce07f637f4996168dc1c879b626706a56cc6871fa317455fbc8fa632b27 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:937cdce07f637f4996168dc1c879b626706a56cc6871fa317455fbc8fa632b27 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 0dc6bb24b8922d37d6c3a269617e9612696d9ffd676102bb5315848b3091770e 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=0dc6bb24b8922d37d6c3a269617e9612696d9ffd676102bb5315848b3091770e 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=21, curl_handle=0x5608c11cd8e0 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdbacf23900 20 run: stack=0x5608c0d5b650 is io blocked 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.065+0000 7fdb067fc640 20 link_request req_data=0x5608c10a1a20 req_data->id=21, curl_handle=0x5608c11cd8e0 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.066+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1201240:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.066+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1201240:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.066+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1201240:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.066+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1201240:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.066+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c12042e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.066+0000 7fdbacf23900 15 stack 0x5608c11c5f10 end 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.066+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is done 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.066+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.066+0000 7fdbacf23900 20 collect(): s=0x5608c0d61b20 stack=0x5608c11c5f10 is complete 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.066+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.066+0000 7fdbacf23900 10 trimming bilogs for bucket=:dipepu-23[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28]) markers=[,,,,,,,,00000000003.16.7,,], shards=11 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.066+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11d2120:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.066+0000 7fdbacf23900 10 trimming bilog shard 8 of :dipepu-23[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28]) at marker 00000000003.16.7 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.066+0000 7fdbacf23900 20 run: stack=0x5608c0d61b20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.066+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1205650:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.066+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1205650:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.066+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28.8 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.066+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is io blocked 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.067+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1205650:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.067+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1205650:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.067+0000 7fdbacf23900 15 stack 0x5608c11c5f10 end 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.067+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.067+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is done 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.067+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11d2120:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.067+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.067+0000 7fdbacf23900 20 trim: trimmed bucket instance dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.067+0000 7fdbacf23900 15 stack 0x5608c0d61b20 end 2026-03-20T11:46:50.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.067+0000 7fdbacf23900 20 run: stack=0x5608c0d61b20 is done 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.067+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.067+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.067+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.067+0000 7fdbacf23900 4 starting trim on bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.067+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c1205650:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.067+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c1205650:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.067+0000 7fdbacf23900 20 enqueued request req=0x5608c11d6400 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.067+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.067+0000 7fdbacf23900 20 req: 0x5608c11d6400 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.067+0000 7fdbacf23900 20 run: stack=0x5608c0d61b20 is io blocked 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.067+0000 7fdb5b7ee640 20 dequeued request req=0x5608c11d6400 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.067+0000 7fdb5b7ee640 20 RGWWQ: empty 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.067+0000 7fdb5b7ee640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 : miss 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.067+0000 7fdb5b7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdb5b7ee640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdb5b7ee640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 info.flags=0x17 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdb5b7ee640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 to cache LRU end 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdb5b7ee640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdb5b7ee640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdb5b7ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdb5b7ee640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-4 : miss 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdb5b7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdb5b7ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdb5b7ee640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-4 info.flags=0x0 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdb5b7ee640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-4 to cache LRU end 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdb5b7ee640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-4 : miss 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdb5b7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdb5b7ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdb5b7ee640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-4 info.flags=0x0 2026-03-20T11:46:50.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdb5b7ee640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-4 to cache LRU end 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdb5b7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5): adding source pipe: {s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdb5b7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5): adding dest pipe: {s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdb5b7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdb5b7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdb5b7ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c1205650:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c1205650:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c1205650:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c1205650:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c12042e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 20 run: stack=0x5608c0d61b20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1201240:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1201240:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 0ef2a9430b4a9cce708b15b9574cdbe68568fa7ad6cf40221847a37758ee4936 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:0ef2a9430b4a9cce708b15b9574cdbe68568fa7ad6cf40221847a37758ee4936 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = e648cdaa202dfb77e942e01afd00d4884b62ac4d1ce25d07fcaf15095a6ec1b2 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=e648cdaa202dfb77e942e01afd00d4884b62ac4d1ce25d07fcaf15095a6ec1b2 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=22, curl_handle=0x5608c0d60ac0 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is io blocked 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.068+0000 7fdb067fc640 20 link_request req_data=0x5608c11d13d0 req_data->id=22, curl_handle=0x5608c0d60ac0 2026-03-20T11:46:50.067 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:50.068+0000 7fd005a59640 -1 req 8737207709359710868 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11cc360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11cc360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11cc360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11cc360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c0d6e300:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 15 stack 0x5608c11c5dd0 end 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is done 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 collect(): s=0x5608c10a18e0 stack=0x5608c11c5dd0 is complete 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 10 trimming bilogs for bucket=:dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) markers=[00000000004.46.6,00000000002.37.1,00000000004.60.6,00000000002.19.1,00000000004.18.6,00000000002.57.1,00000000002.32.1,00000000004.68.6,00000000002.43.1,00000000002.85.1,00000000002.30.1], shards=11 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123fa70:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 10 trimming bilog shard 0 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.46.6 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 10 trimming bilog shard 1 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.37.1 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 10 trimming bilog shard 2 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.60.6 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 10 trimming bilog shard 3 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.19.1 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 10 trimming bilog shard 4 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.18.6 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 10 trimming bilog shard 5 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.57.1 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 10 trimming bilog shard 6 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.32.1 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 10 trimming bilog shard 7 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.68.6 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 10 trimming bilog shard 8 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.43.1 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 10 trimming bilog shard 9 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.85.1 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 10 trimming bilog shard 10 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.30.1 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c12325c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c11cc360:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 cr:s=0x5608c11cbb70:op=0x5608c11d1e40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1231c60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c123d960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 cr:s=0x5608c1243f30:op=0x5608c12361f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 cr:s=0x5608c1200330:op=0x5608c0d5e8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 cr:s=0x5608c1244450:op=0x5608c11d3b80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 cr:s=0x5608c1242cc0:op=0x5608c1240700:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c1238e60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 cr:s=0x5608c1268ab0:op=0x5608c123c230:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c12325c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.0 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is io blocked 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c11cc360:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.1 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 run: stack=0x5608c11c55c0 is io blocked 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 cr:s=0x5608c11cbb70:op=0x5608c11d1e40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.2 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 run: stack=0x5608c11cbb70 is io blocked 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1231c60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.3 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is io blocked 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c123d960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.4 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 run: stack=0x5608c11d6650 is io blocked 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 cr:s=0x5608c1243f30:op=0x5608c12361f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.069+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.5 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.070+0000 7fdbacf23900 20 run: stack=0x5608c1243f30 is io blocked 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.070+0000 7fdbacf23900 20 cr:s=0x5608c1200330:op=0x5608c0d5e8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.070+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.6 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.070+0000 7fdbacf23900 20 run: stack=0x5608c1200330 is io blocked 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.070+0000 7fdbacf23900 20 cr:s=0x5608c1244450:op=0x5608c11d3b80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.070+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.7 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.070+0000 7fdbacf23900 20 run: stack=0x5608c1244450 is io blocked 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.070+0000 7fdbacf23900 20 cr:s=0x5608c1242cc0:op=0x5608c1240700:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.070+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.8 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.070+0000 7fdbacf23900 20 run: stack=0x5608c1242cc0 is io blocked 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.070+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c1238e60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.070+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.9 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.070+0000 7fdbacf23900 20 run: stack=0x5608c120c460 is io blocked 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.070+0000 7fdbacf23900 20 cr:s=0x5608c1268ab0:op=0x5608c123c230:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.070+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.10 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.070+0000 7fdbacf23900 20 run: stack=0x5608c1268ab0 is io blocked 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 cr:s=0x5608c11cbb70:op=0x5608c11d1e40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 cr:s=0x5608c11cbb70:op=0x5608c11d1e40:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 15 stack 0x5608c11cbb70 end 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 run: stack=0x5608c11cbb70 is done 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123fa70:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c11cc360:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c11cc360:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 15 stack 0x5608c11c55c0 end 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 run: stack=0x5608c11c55c0 is done 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123fa70:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c12325c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c12325c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 15 stack 0x5608c11c5dd0 end 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is done 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123fa70:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1231c60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1231c60:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 15 stack 0x5608c11d3250 end 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is done 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123fa70:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c1238e60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c1238e60:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 15 stack 0x5608c120c460 end 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 run: stack=0x5608c120c460 is done 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123fa70:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.071+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 cr:s=0x5608c1200330:op=0x5608c0d5e8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 cr:s=0x5608c1200330:op=0x5608c0d5e8d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 15 stack 0x5608c1200330 end 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 run: stack=0x5608c1200330 is done 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123fa70:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 cr:s=0x5608c1268ab0:op=0x5608c123c230:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 cr:s=0x5608c1268ab0:op=0x5608c123c230:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 15 stack 0x5608c1268ab0 end 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 run: stack=0x5608c1268ab0 is done 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123fa70:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 cr:s=0x5608c1243f30:op=0x5608c12361f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 cr:s=0x5608c1243f30:op=0x5608c12361f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 15 stack 0x5608c1243f30 end 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 run: stack=0x5608c1243f30 is done 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 cr:s=0x5608c1244450:op=0x5608c11d3b80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 cr:s=0x5608c1244450:op=0x5608c11d3b80:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 15 stack 0x5608c1244450 end 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 run: stack=0x5608c1244450 is done 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123fa70:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 cr:s=0x5608c1242cc0:op=0x5608c1240700:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 cr:s=0x5608c1242cc0:op=0x5608c1240700:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 15 stack 0x5608c1242cc0 end 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 run: stack=0x5608c1242cc0 is done 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c123d960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c123d960:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 15 stack 0x5608c11d6650 end 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 run: stack=0x5608c11d6650 is done 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123fa70:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123fa70:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123fa70:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 trim: trimmed bucket instance dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 15 stack 0x5608c10a18e0 end 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is done 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 cr:s=0x5608c1268ab0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 4 starting trim on bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 cr:s=0x5608c1268ab0:op=0x5608c123d080:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 cr:s=0x5608c1268ab0:op=0x5608c123d080:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 enqueued request req=0x5608c11d1130 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 req: 0x5608c11d1130 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdbacf23900 20 run: stack=0x5608c1268ab0 is io blocked 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdb5afed640 20 dequeued request req=0x5608c11d1130 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdb5afed640 20 RGWWQ: empty 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdb5afed640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 : miss 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdb5afed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdb5afed640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdb5afed640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 info.flags=0x17 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdb5afed640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 to cache LRU end 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdb5afed640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdb5afed640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdb5afed640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdb5afed640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-53 : miss 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.072+0000 7fdb5afed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdb5afed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdb5afed640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-53 info.flags=0x0 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdb5afed640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-53 to cache LRU end 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdb5afed640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-53 : miss 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdb5afed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdb5afed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdb5afed640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-53 info.flags=0x0 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdb5afed640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-53 to cache LRU end 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdb5afed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13): adding source pipe: {s={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdb5afed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13): adding dest pipe: {s={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdb5afed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdb5afed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdb5afed640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 20 cr:s=0x5608c1268ab0:op=0x5608c123d080:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 20 cr:s=0x5608c1268ab0:op=0x5608c123d080:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 20 cr:s=0x5608c1268ab0:op=0x5608c123d080:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 20 cr:s=0x5608c1268ab0:op=0x5608c123d080:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 20 cr:s=0x5608c1268ab0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 20 cr:s=0x5608c1200330:op=0x5608c0d6e300:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 20 cr:s=0x5608c1268ab0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 20 run: stack=0x5608c1268ab0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 20 cr:s=0x5608c1200330:op=0x5608c120cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 20 cr:s=0x5608c1200330:op=0x5608c120cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.070 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 5003923cad8ef52a84c4761d6d1cfa09bf2a9fa96681eabb55c1f02b5c23eaa3 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr:5003923cad8ef52a84c4761d6d1cfa09bf2a9fa96681eabb55c1f02b5c23eaa3 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 00ee66f3299027e7c445429aadec45f583621bb755550f3f18d61ce03f4a2212 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=00ee66f3299027e7c445429aadec45f583621bb755550f3f18d61ce03f4a2212 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=23, curl_handle=0x5608c0d5bb20 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdbacf23900 20 run: stack=0x5608c1200330 is io blocked 2026-03-20T11:46:50.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.073+0000 7fdb067fc640 20 link_request req_data=0x5608c11cbe70 req_data->id=23, curl_handle=0x5608c0d5bb20 2026-03-20T11:46:50.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c12082f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c12082f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:50.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c0d614d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdbacf23900 15 stack 0x5608c0d6dec0 end 2026-03-20T11:46:50.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is done 2026-03-20T11:46:50.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdbacf23900 20 collect(): s=0x5608c11c4c90 stack=0x5608c0d6dec0 is complete 2026-03-20T11:46:50.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdbacf23900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:50.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdbacf23900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:50.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:50.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdbacf23900 15 stack 0x5608c11c4c90 end 2026-03-20T11:46:50.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdbacf23900 20 stack->operate() returned ret=-22 2026-03-20T11:46:50.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is done 2026-03-20T11:46:50.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdbacf23900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:50.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdbacf23900 4 starting trim on bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:46:50.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11cc360:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11cc360:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdbacf23900 20 enqueued request req=0x5608c11c67c0 2026-03-20T11:46:50.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdbacf23900 20 req: 0x5608c11c67c0 2026-03-20T11:46:50.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is io blocked 2026-03-20T11:46:50.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdb5a7ec640 20 dequeued request req=0x5608c11c67c0 2026-03-20T11:46:50.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdb5a7ec640 20 RGWWQ: empty 2026-03-20T11:46:50.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdb5a7ec640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 : miss 2026-03-20T11:46:50.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdb5a7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdb5a7ec640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:50.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdb5a7ec640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 info.flags=0x17 2026-03-20T11:46:50.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdb5a7ec640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 to cache LRU end 2026-03-20T11:46:50.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdb5a7ec640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdb5a7ec640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdb5a7ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:46:50.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdb5a7ec640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-14 : miss 2026-03-20T11:46:50.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.102+0000 7fdb5a7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdb5a7ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdb5a7ec640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-14 info.flags=0x0 2026-03-20T11:46:50.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdb5a7ec640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-14 to cache LRU end 2026-03-20T11:46:50.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdb5a7ec640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-14 : miss 2026-03-20T11:46:50.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdb5a7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdb5a7ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdb5a7ec640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-14 info.flags=0x0 2026-03-20T11:46:50.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdb5a7ec640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-14 to cache LRU end 2026-03-20T11:46:50.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdb5a7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19): adding source pipe: {s={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdb5a7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19): adding dest pipe: {s={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdb5a7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdb5a7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdb5a7ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11cc360:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11cc360:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11cc360:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11cc360:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c0d614d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c12082f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c12082f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = d0f41849122c9afc287308012b064b46b2411c6102d8f9905029d17b52bd5981 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:d0f41849122c9afc287308012b064b46b2411c6102d8f9905029d17b52bd5981 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 1d8a064cf19007e2824b2b6771874e892891d26dd4f662629b36f295cc347119 2026-03-20T11:46:50.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=1d8a064cf19007e2824b2b6771874e892891d26dd4f662629b36f295cc347119 2026-03-20T11:46:50.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=24, curl_handle=0x5608c11c9480 2026-03-20T11:46:50.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is io blocked 2026-03-20T11:46:50.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.103+0000 7fdb067fc640 20 link_request req_data=0x5608c1208920 req_data->id=24, curl_handle=0x5608c11c9480 2026-03-20T11:46:50.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.108+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c0d6b640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.108+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c0d6b640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.108+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c0d6b640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.108+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c0d6b640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.108+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c11d5690:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.108+0000 7fdbacf23900 15 stack 0x5608c0d5b650 end 2026-03-20T11:46:50.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.108+0000 7fdbacf23900 20 run: stack=0x5608c0d5b650 is done 2026-03-20T11:46:50.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.108+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.108+0000 7fdbacf23900 20 collect(): s=0x5608c11d3560 stack=0x5608c0d5b650 is complete 2026-03-20T11:46:50.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.108+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.108+0000 7fdbacf23900 10 trimming bilogs for bucket=:dipepu-19[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24]) markers=[,,,,,,,,00000000003.27.6,,], shards=11 2026-03-20T11:46:50.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.108+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d6b640:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.108+0000 7fdbacf23900 10 trimming bilog shard 8 of :dipepu-19[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24]) at marker 00000000003.27.6 2026-03-20T11:46:50.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.108+0000 7fdbacf23900 20 run: stack=0x5608c11d3560 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.108+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c123c230:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.108+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c123c230:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.108+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24.8 2026-03-20T11:46:50.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.108+0000 7fdbacf23900 20 run: stack=0x5608c0d5b650 is io blocked 2026-03-20T11:46:50.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.109+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c123c230:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.109+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c123c230:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.109+0000 7fdbacf23900 15 stack 0x5608c0d5b650 end 2026-03-20T11:46:50.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.109+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.109+0000 7fdbacf23900 20 run: stack=0x5608c0d5b650 is done 2026-03-20T11:46:50.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.109+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c0d6b640:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.109+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.109+0000 7fdbacf23900 20 trim: trimmed bucket instance dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:46:50.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.109+0000 7fdbacf23900 15 stack 0x5608c11d3560 end 2026-03-20T11:46:50.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.109+0000 7fdbacf23900 20 run: stack=0x5608c11d3560 is done 2026-03-20T11:46:50.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.109+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.109+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.109+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.109+0000 7fdbacf23900 4 starting trim on bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:46:50.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.109+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.109+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.109+0000 7fdbacf23900 20 enqueued request req=0x5608c0d5c050 2026-03-20T11:46:50.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.109+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.109+0000 7fdbacf23900 20 req: 0x5608c0d5c050 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.109+0000 7fdbacf23900 20 run: stack=0x5608c11d3560 is io blocked 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.109+0000 7fdb59feb640 20 dequeued request req=0x5608c0d5c050 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.109+0000 7fdb59feb640 20 RGWWQ: empty 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.109+0000 7fdb59feb640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 : miss 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.109+0000 7fdb59feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1201240:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb59feb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb59feb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 info.flags=0x17 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb59feb640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 to cache LRU end 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb59feb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1201240:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb59feb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c12042e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 15 stack 0x5608c11c5f10 end 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is done 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 collect(): s=0x5608c0d61b20 stack=0x5608c11c5f10 is complete 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 cr:s=0x5608c0d61b20:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb59feb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 15 stack 0x5608c0d61b20 end 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 stack->operate() returned ret=-22 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 run: stack=0x5608c0d61b20 is done 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb59feb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-36 : miss 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb59feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 4 starting trim on bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11cc360:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11cc360:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 enqueued request req=0x5608c11d6400 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 req: 0x5608c11d6400 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is io blocked 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb597ea640 20 dequeued request req=0x5608c11d6400 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb597ea640 20 RGWWQ: empty 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb597ea640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 : miss 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb597ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb59feb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb59feb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-36 info.flags=0x0 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb59feb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-36 to cache LRU end 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb59feb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-36 : miss 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb59feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb597ea640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb597ea640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 info.flags=0x17 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb597ea640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 to cache LRU end 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb597ea640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb597ea640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb597ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb597ea640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-22 : miss 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb597ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb59feb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb59feb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-36 info.flags=0x0 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb59feb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-36 to cache LRU end 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb59feb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41): adding source pipe: {s={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb59feb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41): adding dest pipe: {s={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb59feb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb59feb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdb59feb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c0d6b640:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 run: stack=0x5608c11d3560 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c126aad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c126aad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.110+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 07d2dc7db89ebf47d950378bbc2cdf0002dcf3c826d4a37334bbae20e33bcc35 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:07d2dc7db89ebf47d950378bbc2cdf0002dcf3c826d4a37334bbae20e33bcc35 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 7a640c64477496730e09369bd6fc3ded99e9cc33d715c04d415df08a77773b89 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=7a640c64477496730e09369bd6fc3ded99e9cc33d715c04d415df08a77773b89 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=25, curl_handle=0x5608c0d60ac0 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 20 run: stack=0x5608c0d5b650 is io blocked 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdb067fc640 20 link_request req_data=0x5608c11d13d0 req_data->id=25, curl_handle=0x5608c0d60ac0 2026-03-20T11:46:50.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdb597ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdb597ea640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-22 info.flags=0x0 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdb597ea640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-22 to cache LRU end 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdb597ea640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-22 : miss 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdb597ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdb597ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdb597ea640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-22 info.flags=0x0 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdb597ea640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-22 to cache LRU end 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdb597ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27): adding source pipe: {s={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdb597ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27): adding dest pipe: {s={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdb597ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdb597ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdb597ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11cc360:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11cc360:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11cc360:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11cc360:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c12042e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c11cc360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c11cc360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = f493b50c6ab5fe30273749511c19fb163dd572149d77d34297fde2dbdd19e7f6 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:f493b50c6ab5fe30273749511c19fb163dd572149d77d34297fde2dbdd19e7f6 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = d9b9b104b0c6395c5702a23affd7e2d19cfc5031f77002860d47e5fef1e27718 2026-03-20T11:46:50.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=d9b9b104b0c6395c5702a23affd7e2d19cfc5031f77002860d47e5fef1e27718 2026-03-20T11:46:50.111 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.111 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.111 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.111 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=26, curl_handle=0x5608c11cd8e0 2026-03-20T11:46:50.111 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdbacf23900 20 run: stack=0x5608c120c460 is io blocked 2026-03-20T11:46:50.111 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.111+0000 7fdb067fc640 20 link_request req_data=0x5608c10a1a20 req_data->id=26, curl_handle=0x5608c11cd8e0 2026-03-20T11:46:50.112 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c1200330:op=0x5608c120cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c1200330:op=0x5608c120cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c1200330:op=0x5608c120cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c1200330:op=0x5608c120cce0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c1200330:op=0x5608c0d6e300:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 15 stack 0x5608c1200330 end 2026-03-20T11:46:50.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 run: stack=0x5608c1200330 is done 2026-03-20T11:46:50.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c1268ab0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 collect(): s=0x5608c1268ab0 stack=0x5608c1200330 is complete 2026-03-20T11:46:50.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 10 trimming bilogs for bucket=:dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) markers=[00000000004.65.1,00000000002.83.1,00000000004.51.1,00000000002.21.1,00000000004.74.1,00000000002.72.1,00000000002.84.1,00000000004.73.1,00000000002.22.1,00000000002.50.1,00000000002.35.1], shards=11 2026-03-20T11:46:50.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c1268ab0:op=0x5608c0d6e300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 10 trimming bilog shard 0 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.65.1 2026-03-20T11:46:50.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 10 trimming bilog shard 1 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.83.1 2026-03-20T11:46:50.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 10 trimming bilog shard 2 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.51.1 2026-03-20T11:46:50.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 10 trimming bilog shard 3 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.21.1 2026-03-20T11:46:50.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 10 trimming bilog shard 4 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.74.1 2026-03-20T11:46:50.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 10 trimming bilog shard 5 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.72.1 2026-03-20T11:46:50.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 10 trimming bilog shard 6 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.84.1 2026-03-20T11:46:50.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 10 trimming bilog shard 7 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.73.1 2026-03-20T11:46:50.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 10 trimming bilog shard 8 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.22.1 2026-03-20T11:46:50.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 10 trimming bilog shard 9 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.50.1 2026-03-20T11:46:50.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 10 trimming bilog shard 10 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.35.1 2026-03-20T11:46:50.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 run: stack=0x5608c1268ab0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c1200330:op=0x5608c1241f90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c123a450:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c1240060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c120ca00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c11cbb70:op=0x5608c11d4600:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c11bee50:op=0x5608c123cf10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123b7d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c12318c0:op=0x5608c11d1c60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c12364c0:op=0x5608c123e6a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1233f20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c1237310:op=0x5608c1240970:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c1200330:op=0x5608c1241f90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.0 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 run: stack=0x5608c1200330 is io blocked 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c123a450:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.1 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is io blocked 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c1240060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.2 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is io blocked 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c120ca00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.3 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 run: stack=0x5608c11c55c0 is io blocked 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c11cbb70:op=0x5608c11d4600:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.4 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 run: stack=0x5608c11cbb70 is io blocked 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c11bee50:op=0x5608c123cf10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.5 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 run: stack=0x5608c11bee50 is io blocked 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123b7d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.6 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is io blocked 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c12318c0:op=0x5608c11d1c60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.7 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 run: stack=0x5608c12318c0 is io blocked 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c12364c0:op=0x5608c123e6a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.8 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 run: stack=0x5608c12364c0 is io blocked 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1233f20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.9 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 run: stack=0x5608c11d6650 is io blocked 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 cr:s=0x5608c1237310:op=0x5608c1240970:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.10 2026-03-20T11:46:50.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.116+0000 7fdbacf23900 20 run: stack=0x5608c1237310 is io blocked 2026-03-20T11:46:50.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c120ca00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c120ca00:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 15 stack 0x5608c11c55c0 end 2026-03-20T11:46:50.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 run: stack=0x5608c11c55c0 is done 2026-03-20T11:46:50.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c1268ab0:op=0x5608c0d6e300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 run: stack=0x5608c1268ab0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1233f20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1233f20:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 15 stack 0x5608c11d6650 end 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 run: stack=0x5608c11d6650 is done 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c123a450:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c123a450:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 15 stack 0x5608c11d3250 end 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is done 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c11bee50:op=0x5608c123cf10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c11bee50:op=0x5608c123cf10:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 15 stack 0x5608c11bee50 end 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 run: stack=0x5608c11bee50 is done 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c1240060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c1240060:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 15 stack 0x5608c11c5dd0 end 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is done 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c1237310:op=0x5608c1240970:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c1237310:op=0x5608c1240970:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 15 stack 0x5608c1237310 end 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 run: stack=0x5608c1237310 is done 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c1200330:op=0x5608c1241f90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c1200330:op=0x5608c1241f90:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 15 stack 0x5608c1200330 end 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 run: stack=0x5608c1200330 is done 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c12318c0:op=0x5608c11d1c60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c12318c0:op=0x5608c11d1c60:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 15 stack 0x5608c12318c0 end 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 run: stack=0x5608c12318c0 is done 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c1268ab0:op=0x5608c0d6e300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c1268ab0:op=0x5608c0d6e300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c1268ab0:op=0x5608c0d6e300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c1268ab0:op=0x5608c0d6e300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c1268ab0:op=0x5608c0d6e300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c1268ab0:op=0x5608c0d6e300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c1268ab0:op=0x5608c0d6e300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 run: stack=0x5608c1268ab0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c12364c0:op=0x5608c123e6a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c12364c0:op=0x5608c123e6a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 15 stack 0x5608c12364c0 end 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 run: stack=0x5608c12364c0 is done 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c1268ab0:op=0x5608c0d6e300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 run: stack=0x5608c1268ab0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123b7d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123b7d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 15 stack 0x5608c10a18e0 end 2026-03-20T11:46:50.116 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is done 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c1268ab0:op=0x5608c0d6e300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 run: stack=0x5608c1268ab0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c11cbb70:op=0x5608c11d4600:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c11cbb70:op=0x5608c11d4600:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 15 stack 0x5608c11cbb70 end 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 run: stack=0x5608c11cbb70 is done 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c1268ab0:op=0x5608c0d6e300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c1268ab0:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 trim: trimmed bucket instance dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 15 stack 0x5608c1268ab0 end 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 run: stack=0x5608c1268ab0 is done 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c1237310:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 4 starting trim on bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c1237310:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 cr:s=0x5608c1237310:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 enqueued request req=0x5608c1201fe0 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 req: 0x5608c1201fe0 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdbacf23900 20 run: stack=0x5608c1237310 is io blocked 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdb58fe9640 20 dequeued request req=0x5608c1201fe0 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdb58fe9640 20 RGWWQ: empty 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdb58fe9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 : miss 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.118+0000 7fdb58fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.117 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:50.119+0000 7fd010a6f640 -1 req 10078582171867811166 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdb58fe9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdb58fe9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 info.flags=0x17 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdb58fe9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 to cache LRU end 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdb58fe9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdb58fe9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdb58fe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdb58fe9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-25 : miss 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdb58fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdb58fe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdb58fe9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-25 info.flags=0x0 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdb58fe9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-25 to cache LRU end 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdb58fe9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-25 : miss 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdb58fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdb58fe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdb58fe9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-25 info.flags=0x0 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdb58fe9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-25 to cache LRU end 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdb58fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30): adding source pipe: {s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdb58fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30): adding dest pipe: {s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdb58fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdb58fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdb58fe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdbacf23900 20 cr:s=0x5608c1237310:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdbacf23900 20 cr:s=0x5608c1237310:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdbacf23900 20 cr:s=0x5608c1237310:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdbacf23900 20 cr:s=0x5608c1237310:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdbacf23900 20 cr:s=0x5608c1237310:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c0d6e300:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdbacf23900 20 cr:s=0x5608c1237310:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdbacf23900 20 run: stack=0x5608c1237310 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1268ab0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1268ab0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.119+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.120+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.120+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.120+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.120+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.120+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = ba9ec1ea95e9e1e7eac169b52185d9f364fbd8d03e78e3ea42f8cc2f9c5609ca 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.120+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:ba9ec1ea95e9e1e7eac169b52185d9f364fbd8d03e78e3ea42f8cc2f9c5609ca 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.120+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.120+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.120+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.120+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.120+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 4ca97b480ce165e23ae96c1caedf780b3362acc41231102f86fb6265cc8de2d6 2026-03-20T11:46:50.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.120+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=4ca97b480ce165e23ae96c1caedf780b3362acc41231102f86fb6265cc8de2d6 2026-03-20T11:46:50.119 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.120+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.119 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.120+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.119 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.120+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.119 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.120+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=27, curl_handle=0x5608c0d5bb20 2026-03-20T11:46:50.119 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.120+0000 7fdbacf23900 20 run: stack=0x5608c11d6650 is io blocked 2026-03-20T11:46:50.119 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.120+0000 7fdb067fc640 20 link_request req_data=0x5608c11cbe70 req_data->id=27, curl_handle=0x5608c0d5bb20 2026-03-20T11:46:50.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.146+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c12082f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.146+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c12082f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.146+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c12082f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.146+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c12082f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.146+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c0d614d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.146+0000 7fdbacf23900 15 stack 0x5608c0d6dec0 end 2026-03-20T11:46:50.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.146+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is done 2026-03-20T11:46:50.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.146+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.146+0000 7fdbacf23900 20 collect(): s=0x5608c11c4c90 stack=0x5608c0d6dec0 is complete 2026-03-20T11:46:50.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.146+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.146+0000 7fdbacf23900 10 trimming bilogs for bucket=:dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) markers=[,,,,,00000000002.32.11,,,00000000003.22.6,00000000011.52.6,], shards=11 2026-03-20T11:46:50.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.146+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c1238e60:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.146+0000 7fdbacf23900 10 trimming bilog shard 5 of :dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) at marker 00000000002.32.11 2026-03-20T11:46:50.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.146+0000 7fdbacf23900 10 trimming bilog shard 8 of :dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) at marker 00000000003.22.6 2026-03-20T11:46:50.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.146+0000 7fdbacf23900 10 trimming bilog shard 9 of :dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) at marker 00000000011.52.6 2026-03-20T11:46:50.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.146+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.146+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1244080:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.146+0000 7fdbacf23900 20 cr:s=0x5608c11bee50:op=0x5608c1231090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.146+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c12699f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.146+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1244080:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.146+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19.5 2026-03-20T11:46:50.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.146+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is io blocked 2026-03-20T11:46:50.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.146+0000 7fdbacf23900 20 cr:s=0x5608c11bee50:op=0x5608c1231090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.146+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19.8 2026-03-20T11:46:50.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.146+0000 7fdbacf23900 20 run: stack=0x5608c11bee50 is io blocked 2026-03-20T11:46:50.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.146+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c12699f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.146+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19.9 2026-03-20T11:46:50.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.146+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is io blocked 2026-03-20T11:46:50.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.147+0000 7fdbacf23900 20 cr:s=0x5608c11bee50:op=0x5608c1231090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.147+0000 7fdbacf23900 20 cr:s=0x5608c11bee50:op=0x5608c1231090:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.147+0000 7fdbacf23900 15 stack 0x5608c11bee50 end 2026-03-20T11:46:50.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.147+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.147+0000 7fdbacf23900 20 run: stack=0x5608c11bee50 is done 2026-03-20T11:46:50.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.147+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c1238e60:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.147+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c12699f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c12699f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdbacf23900 15 stack 0x5608c11c5dd0 end 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is done 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c1238e60:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1244080:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1244080:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdbacf23900 15 stack 0x5608c0d6dec0 end 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is done 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c1238e60:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdbacf23900 20 trim: trimmed bucket instance dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdbacf23900 15 stack 0x5608c11c4c90 end 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is done 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdbacf23900 4 starting trim on bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c123e330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c123e330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdbacf23900 20 enqueued request req=0x5608c11c67c0 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdbacf23900 20 req: 0x5608c11c67c0 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is io blocked 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdb587e8640 20 dequeued request req=0x5608c11c67c0 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdb587e8640 20 RGWWQ: empty 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdb587e8640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 : miss 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdb587e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdb587e8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdb587e8640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 info.flags=0x17 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdb587e8640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 to cache LRU end 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdb587e8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdb587e8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdb587e8640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdb587e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdb587e8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-35 : miss 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdb587e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdb587e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdb587e8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-35 info.flags=0x0 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdb587e8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-35 to cache LRU end 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdb587e8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-35 : miss 2026-03-20T11:46:50.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.148+0000 7fdb587e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdb587e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdb587e8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-35 info.flags=0x0 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdb587e8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-35 to cache LRU end 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdb587e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40): adding source pipe: {s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdb587e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40): adding dest pipe: {s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdb587e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdb587e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdb587e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c123e330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c123e330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c123e330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c123e330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c0d614d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1236bf0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1236bf0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = f784a1da2cc32eec2c3d11e66b8bf1eea18fe147eb292aae1a8743499c7b6b2b 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:f784a1da2cc32eec2c3d11e66b8bf1eea18fe147eb292aae1a8743499c7b6b2b 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 3dbe596fe99b2be6a6828b86afcfabccb03510c8915207be2847d0a3192d59db 2026-03-20T11:46:50.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=3dbe596fe99b2be6a6828b86afcfabccb03510c8915207be2847d0a3192d59db 2026-03-20T11:46:50.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=28, curl_handle=0x5608c11c9480 2026-03-20T11:46:50.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is io blocked 2026-03-20T11:46:50.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.149+0000 7fdb067fc640 20 link_request req_data=0x5608c1208920 req_data->id=28, curl_handle=0x5608c11c9480 2026-03-20T11:46:50.147 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:50.148+0000 7fd02baa5640 -1 req 10652937906914257638 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:50.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.154+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c11cc360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.154+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c11cc360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.154+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c11cc360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.154+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c11cc360:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.154+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c12042e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.154+0000 7fdbacf23900 15 stack 0x5608c120c460 end 2026-03-20T11:46:50.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.154+0000 7fdbacf23900 20 run: stack=0x5608c120c460 is done 2026-03-20T11:46:50.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.154+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.154+0000 7fdbacf23900 20 collect(): s=0x5608c11c5f10 stack=0x5608c120c460 is complete 2026-03-20T11:46:50.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.154+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.154+0000 7fdbacf23900 10 trimming bilogs for bucket=:dipepu-22[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27]) markers=[,,,,,,,,00000000006.25.9,,], shards=11 2026-03-20T11:46:50.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.154+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11cc360:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.154+0000 7fdbacf23900 10 trimming bilog shard 8 of :dipepu-22[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27]) at marker 00000000006.25.9 2026-03-20T11:46:50.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.154+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.154+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c1231090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.154+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c1231090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.154+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27.8 2026-03-20T11:46:50.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.154+0000 7fdbacf23900 20 run: stack=0x5608c120c460 is io blocked 2026-03-20T11:46:50.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.155+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c126aad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.155+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c126aad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.155+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c126aad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.155+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c126aad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.155+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c0d6b640:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.155+0000 7fdbacf23900 15 stack 0x5608c0d5b650 end 2026-03-20T11:46:50.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.155+0000 7fdbacf23900 20 run: stack=0x5608c0d5b650 is done 2026-03-20T11:46:50.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.155+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.155+0000 7fdbacf23900 20 collect(): s=0x5608c11d3560 stack=0x5608c0d5b650 is complete 2026-03-20T11:46:50.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.155+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.155+0000 7fdbacf23900 10 trimming bilogs for bucket=:dipepu-36[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41]) markers=[,,,,,,,,,,00000000001.34.6], shards=11 2026-03-20T11:46:50.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.155+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c12082f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.155+0000 7fdbacf23900 10 trimming bilog shard 10 of :dipepu-36[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41]) at marker 00000000001.34.6 2026-03-20T11:46:50.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.155+0000 7fdbacf23900 20 run: stack=0x5608c11d3560 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.155+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c12699f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.155+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c12699f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.155+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41.10 2026-03-20T11:46:50.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.155+0000 7fdbacf23900 20 run: stack=0x5608c0d5b650 is io blocked 2026-03-20T11:46:50.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c1231090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c1231090:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 15 stack 0x5608c120c460 end 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 run: stack=0x5608c120c460 is done 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11cc360:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 trim: trimmed bucket instance dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 15 stack 0x5608c11c5f10 end 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is done 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 4 starting trim on bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c126a270:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c126a270:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 enqueued request req=0x5608c11d6400 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 req: 0x5608c11d6400 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is io blocked 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdb6bfff640 20 dequeued request req=0x5608c11d6400 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdb6bfff640 20 RGWWQ: empty 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdb6bfff640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 : miss 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdb6bfff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c12699f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c12699f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 15 stack 0x5608c0d5b650 end 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 run: stack=0x5608c0d5b650 is done 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c12082f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 cr:s=0x5608c11d3560:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 trim: trimmed bucket instance dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 15 stack 0x5608c11d3560 end 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 run: stack=0x5608c11d3560 is done 2026-03-20T11:46:50.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 4 starting trim on bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:50.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c1236030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c1236030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdb6bfff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:50.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 enqueued request req=0x5608c0d5c050 2026-03-20T11:46:50.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 req: 0x5608c0d5c050 2026-03-20T11:46:50.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdbacf23900 20 run: stack=0x5608c0d5b650 is io blocked 2026-03-20T11:46:50.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdb6bfff640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 info.flags=0x17 2026-03-20T11:46:50.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdb6bfff640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 to cache LRU end 2026-03-20T11:46:50.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdb6bfff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdb6bfff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdb57fe7640 20 dequeued request req=0x5608c0d5c050 2026-03-20T11:46:50.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdb57fe7640 20 RGWWQ: empty 2026-03-20T11:46:50.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdb6bfff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:46:50.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdb6bfff640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-26 : miss 2026-03-20T11:46:50.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdb6bfff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdb57fe7640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 : miss 2026-03-20T11:46:50.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.156+0000 7fdb57fe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb6bfff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb6bfff640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-26 info.flags=0x0 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb6bfff640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-26 to cache LRU end 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb6bfff640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-26 : miss 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb6bfff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb57fe7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb57fe7640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 info.flags=0x17 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb57fe7640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 to cache LRU end 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb57fe7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb57fe7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb57fe7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb57fe7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-51 : miss 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb57fe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb6bfff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb6bfff640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-26 info.flags=0x0 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb6bfff640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-26 to cache LRU end 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb57fe7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb57fe7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-51 info.flags=0x0 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb6bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31): adding source pipe: {s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb57fe7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-51 to cache LRU end 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb57fe7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-51 : miss 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb57fe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb6bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31): adding dest pipe: {s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb6bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb6bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb6bfff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c126a270:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c126a270:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c126a270:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c126a270:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c12042e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c0d6b640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c0d6b640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 34ce1c519067ce9711f45e952973d5d3529eff2d8ec946b69870623abdae77d7 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:34ce1c519067ce9711f45e952973d5d3529eff2d8ec946b69870623abdae77d7 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 836b5996843c02021f48889f892ea78803770f755aa3c9dbcdd419cd2a73f5af 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=836b5996843c02021f48889f892ea78803770f755aa3c9dbcdd419cd2a73f5af 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=29, curl_handle=0x5608c0d60ac0 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 run: stack=0x5608c120c460 is io blocked 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb067fc640 20 link_request req_data=0x5608c11d13d0 req_data->id=29, curl_handle=0x5608c0d60ac0 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb57fe7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb57fe7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-51 info.flags=0x0 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb57fe7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-51 to cache LRU end 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb57fe7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11): adding source pipe: {s={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb57fe7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11): adding dest pipe: {s={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb57fe7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb57fe7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdb57fe7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c1236030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c1236030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c1236030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c1236030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c126aad0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 run: stack=0x5608c0d5b650 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c1242ea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c1242ea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.157+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.158+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.158+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.158+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = a209f5219113fc38d605162c2981c97dba31eb081cf84d7643db44a0448fade6 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.158+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:a209f5219113fc38d605162c2981c97dba31eb081cf84d7643db44a0448fade6 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.158+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.158+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.158+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.158+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.158+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 8be8d1d7ef725624444b1c2e1631560e768d4197f32c74fa59e76b29a78a1376 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.158+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=8be8d1d7ef725624444b1c2e1631560e768d4197f32c74fa59e76b29a78a1376 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.158+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.158+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.158+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.158+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=30, curl_handle=0x5608c11cd8e0 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.158+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is io blocked 2026-03-20T11:46:50.157 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.158+0000 7fdb067fc640 20 link_request req_data=0x5608c10a1a20 req_data->id=30, curl_handle=0x5608c11cd8e0 2026-03-20T11:46:50.158 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:50.157+0000 7fd0422d2640 -1 req 13895128305179939361 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1268ab0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1268ab0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c0d6e300:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdbacf23900 15 stack 0x5608c11d6650 end 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdbacf23900 20 run: stack=0x5608c11d6650 is done 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdbacf23900 20 cr:s=0x5608c1237310:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdbacf23900 20 collect(): s=0x5608c1237310 stack=0x5608c11d6650 is complete 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdbacf23900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdbacf23900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdbacf23900 20 cr:s=0x5608c1237310:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdbacf23900 15 stack 0x5608c1237310 end 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdbacf23900 20 stack->operate() returned ret=-22 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdbacf23900 20 run: stack=0x5608c1237310 is done 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdbacf23900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdbacf23900 20 cr:s=0x5608c1237310:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdbacf23900 4 starting trim on bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdbacf23900 20 cr:s=0x5608c1237310:op=0x5608c1236030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdbacf23900 20 cr:s=0x5608c1237310:op=0x5608c1236030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdbacf23900 20 enqueued request req=0x5608c1201fe0 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdbacf23900 20 req: 0x5608c1201fe0 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdbacf23900 20 run: stack=0x5608c1237310 is io blocked 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdb6affd640 20 dequeued request req=0x5608c1201fe0 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdb6affd640 20 RGWWQ: empty 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdb6affd640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 : miss 2026-03-20T11:46:50.158 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdb6affd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdb6affd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdb6affd640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 info.flags=0x17 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdb6affd640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 to cache LRU end 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdb6affd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdb6affd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdb6affd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdb6affd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-55 : miss 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdb6affd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdb6affd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdb6affd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-55 info.flags=0x0 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdb6affd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-55 to cache LRU end 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdb6affd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-55 : miss 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.161+0000 7fdb6affd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdb6affd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdb6affd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-55 info.flags=0x0 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdb6affd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-55 to cache LRU end 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdb6affd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding source pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdb6affd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding dest pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdb6affd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdb6affd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdb6affd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 20 cr:s=0x5608c1237310:op=0x5608c1236030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 20 cr:s=0x5608c1237310:op=0x5608c1236030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 20 cr:s=0x5608c1237310:op=0x5608c1236030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 20 cr:s=0x5608c1237310:op=0x5608c1236030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 20 cr:s=0x5608c1237310:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c0d6e300:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 20 cr:s=0x5608c1237310:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 20 run: stack=0x5608c1237310 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1268ab0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1268ab0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = b4f59e4c14b36036d6f58513119116ff5770dc2c7d77eee7c63085263ddcb623 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:b4f59e4c14b36036d6f58513119116ff5770dc2c7d77eee7c63085263ddcb623 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.159 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.160 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.160 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.160 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 5e0b9db242366fec8836280d7e0e48241753de453c02d3ed257a5c98a6b5afd2 2026-03-20T11:46:50.160 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5e0b9db242366fec8836280d7e0e48241753de453c02d3ed257a5c98a6b5afd2 2026-03-20T11:46:50.160 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.160 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.160 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.160 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=31, curl_handle=0x5608c0d5bb20 2026-03-20T11:46:50.160 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdbacf23900 20 run: stack=0x5608c11d6650 is io blocked 2026-03-20T11:46:50.160 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.162+0000 7fdb067fc640 20 link_request req_data=0x5608c11cbe70 req_data->id=31, curl_handle=0x5608c0d5bb20 2026-03-20T11:46:50.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1236bf0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1236bf0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:50.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c0d614d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdbacf23900 15 stack 0x5608c0d6dec0 end 2026-03-20T11:46:50.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is done 2026-03-20T11:46:50.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdbacf23900 20 collect(): s=0x5608c11c4c90 stack=0x5608c0d6dec0 is complete 2026-03-20T11:46:50.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdbacf23900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:46:50.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdbacf23900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:50.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:50.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdbacf23900 15 stack 0x5608c11c4c90 end 2026-03-20T11:46:50.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdbacf23900 20 stack->operate() returned ret=-22 2026-03-20T11:46:50.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is done 2026-03-20T11:46:50.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdbacf23900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:50.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdbacf23900 4 starting trim on bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:50.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c1236030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c1236030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdbacf23900 20 enqueued request req=0x5608c11c67c0 2026-03-20T11:46:50.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdbacf23900 20 req: 0x5608c11c67c0 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is io blocked 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdb6a7fc640 20 dequeued request req=0x5608c11c67c0 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdb6a7fc640 20 RGWWQ: empty 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdb6a7fc640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 : miss 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdb6a7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdb6a7fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdb6a7fc640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 info.flags=0x17 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdb6a7fc640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 to cache LRU end 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdb6a7fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdb6a7fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdb6a7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdb6a7fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-28 : miss 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.190+0000 7fdb6a7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdb6a7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdb6a7fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-28 info.flags=0x0 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdb6a7fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-28 to cache LRU end 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdb6a7fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-28 : miss 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdb6a7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdb6a7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdb6a7fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-28 info.flags=0x0 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdb6a7fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-28 to cache LRU end 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdb6a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding source pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdb6a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding dest pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdb6a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdb6a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdb6a7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c1236030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c1236030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c1236030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c1236030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c0d614d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1240e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1240e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 7dc0a677e29277b0704f9aa2681083ed5e31d190d5cc82add58dd70943e0072e 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:7dc0a677e29277b0704f9aa2681083ed5e31d190d5cc82add58dd70943e0072e 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 64affbba8c9a9a74c20911cec8ac59e1c9158fcce7d5575503c00832c89e5f62 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=64affbba8c9a9a74c20911cec8ac59e1c9158fcce7d5575503c00832c89e5f62 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=32, curl_handle=0x5608c11c9480 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is io blocked 2026-03-20T11:46:50.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.191+0000 7fdb067fc640 20 link_request req_data=0x5608c1208920 req_data->id=32, curl_handle=0x5608c11c9480 2026-03-20T11:46:50.189 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:50.191+0000 7fcfe9a21640 -1 req 3751564090313143767 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c0d6b640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c0d6b640:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c12042e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdbacf23900 15 stack 0x5608c120c460 end 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdbacf23900 20 run: stack=0x5608c120c460 is done 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdbacf23900 20 collect(): s=0x5608c11c5f10 stack=0x5608c120c460 is complete 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdbacf23900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdbacf23900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdbacf23900 15 stack 0x5608c11c5f10 end 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdbacf23900 20 stack->operate() returned ret=-22 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is done 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdbacf23900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdbacf23900 4 starting trim on bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11cc360:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11cc360:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdbacf23900 20 enqueued request req=0x5608c11d6400 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdbacf23900 20 req: 0x5608c11d6400 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is io blocked 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdb69ffb640 20 dequeued request req=0x5608c11d6400 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdb69ffb640 20 RGWWQ: empty 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdb69ffb640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 : miss 2026-03-20T11:46:50.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdb69ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdb69ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:50.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdb69ffb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 info.flags=0x17 2026-03-20T11:46:50.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdb69ffb640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 to cache LRU end 2026-03-20T11:46:50.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdb69ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdb69ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:50.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdb69ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:50.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdb69ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-31 : miss 2026-03-20T11:46:50.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.199+0000 7fdb69ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdb69ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdb69ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-31 info.flags=0x0 2026-03-20T11:46:50.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdb69ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-31 to cache LRU end 2026-03-20T11:46:50.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdb69ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-31 : miss 2026-03-20T11:46:50.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdb69ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdb69ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdb69ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-31 info.flags=0x0 2026-03-20T11:46:50.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdb69ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-31 to cache LRU end 2026-03-20T11:46:50.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdb69ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding source pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdb69ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding dest pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdb69ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdb69ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdb69ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:50.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11cc360:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11cc360:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11cc360:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11cc360:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c12042e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c0d6b640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c0d6b640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = a7b0e4328aea0bf28ea2a83cab7d5014f8589d58da207c833c304d79027ef507 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:a7b0e4328aea0bf28ea2a83cab7d5014f8589d58da207c833c304d79027ef507 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 6e6b40049db3daf9d21d6c489e4571804bf1736c96f7baefa71a95e055626a08 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=6e6b40049db3daf9d21d6c489e4571804bf1736c96f7baefa71a95e055626a08 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=33, curl_handle=0x5608c0d60ac0 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdbacf23900 20 run: stack=0x5608c120c460 is io blocked 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.200+0000 7fdb067fc640 20 link_request req_data=0x5608c11d13d0 req_data->id=33, curl_handle=0x5608c0d60ac0 2026-03-20T11:46:50.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c1242ea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.198 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:50.200+0000 7fd044ad7640 -1 req 2659391679287709838 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:50.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c1242ea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c1242ea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c1242ea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c126aad0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 15 stack 0x5608c11c5dd0 end 2026-03-20T11:46:50.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is done 2026-03-20T11:46:50.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 collect(): s=0x5608c0d5b650 stack=0x5608c11c5dd0 is complete 2026-03-20T11:46:50.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 10 trimming bilogs for bucket=:dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) markers=[00000000004.41.6,00000000002.16.1,00000000004.28.6,00000000002.34.1,00000000004.45.6,00000000002.40.1,00000000002.39.1,00000000004.52.6,00000000002.73.1,00000000002.41.1,00000000002.74.1], shards=11 2026-03-20T11:46:50.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c12082f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 10 trimming bilog shard 0 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000004.41.6 2026-03-20T11:46:50.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 10 trimming bilog shard 1 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.16.1 2026-03-20T11:46:50.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 10 trimming bilog shard 2 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000004.28.6 2026-03-20T11:46:50.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 10 trimming bilog shard 3 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.34.1 2026-03-20T11:46:50.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 10 trimming bilog shard 4 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000004.45.6 2026-03-20T11:46:50.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 10 trimming bilog shard 5 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.40.1 2026-03-20T11:46:50.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 10 trimming bilog shard 6 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.39.1 2026-03-20T11:46:50.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 10 trimming bilog shard 7 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000004.52.6 2026-03-20T11:46:50.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 10 trimming bilog shard 8 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.73.1 2026-03-20T11:46:50.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 10 trimming bilog shard 9 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.41.1 2026-03-20T11:46:50.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 10 trimming bilog shard 10 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.74.1 2026-03-20T11:46:50.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 run: stack=0x5608c0d5b650 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c12699f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 cr:s=0x5608c11bee50:op=0x5608c123a4c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c126c0b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 cr:s=0x5608c1200330:op=0x5608c123cf10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c120c840:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11d3c90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 cr:s=0x5608c12325c0:op=0x5608c1240060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c1244080:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 cr:s=0x5608c123b620:op=0x5608c123ddc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 cr:s=0x5608c1240ba0:op=0x5608c123e5c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 cr:s=0x5608c1201890:op=0x5608c126dd20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c12699f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.0 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is io blocked 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 cr:s=0x5608c11bee50:op=0x5608c123a4c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.1 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 run: stack=0x5608c11bee50 is io blocked 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c126c0b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.2 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is io blocked 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 cr:s=0x5608c1200330:op=0x5608c123cf10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.3 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 run: stack=0x5608c1200330 is io blocked 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c120c840:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.4 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 run: stack=0x5608c11c55c0 is io blocked 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11d3c90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.5 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is io blocked 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 cr:s=0x5608c12325c0:op=0x5608c1240060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.201+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.6 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.202+0000 7fdbacf23900 20 run: stack=0x5608c12325c0 is io blocked 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.202+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c1244080:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.202+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.7 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.202+0000 7fdbacf23900 20 run: stack=0x5608c0d5f270 is io blocked 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.202+0000 7fdbacf23900 20 cr:s=0x5608c123b620:op=0x5608c123ddc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.202+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.8 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.202+0000 7fdbacf23900 20 run: stack=0x5608c123b620 is io blocked 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.202+0000 7fdbacf23900 20 cr:s=0x5608c1240ba0:op=0x5608c123e5c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.202+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.9 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.202+0000 7fdbacf23900 20 run: stack=0x5608c1240ba0 is io blocked 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.202+0000 7fdbacf23900 20 cr:s=0x5608c1201890:op=0x5608c126dd20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.202+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.10 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.202+0000 7fdbacf23900 20 run: stack=0x5608c1201890 is io blocked 2026-03-20T11:46:50.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.204+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1268ab0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.204+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1268ab0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.204+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1268ab0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.204+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1268ab0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.204+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c0d6e300:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.204+0000 7fdbacf23900 15 stack 0x5608c11d6650 end 2026-03-20T11:46:50.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.204+0000 7fdbacf23900 20 run: stack=0x5608c11d6650 is done 2026-03-20T11:46:50.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.204+0000 7fdbacf23900 20 cr:s=0x5608c1237310:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.204+0000 7fdbacf23900 20 collect(): s=0x5608c1237310 stack=0x5608c11d6650 is complete 2026-03-20T11:46:50.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.204+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.204+0000 7fdbacf23900 10 trimming bilogs for bucket=:dipepu-55[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15]) markers=[,,,,,00000000006.98.6,,,,,], shards=11 2026-03-20T11:46:50.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.204+0000 7fdbacf23900 20 cr:s=0x5608c1237310:op=0x5608c123c1d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.204+0000 7fdbacf23900 10 trimming bilog shard 5 of :dipepu-55[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15]) at marker 00000000006.98.6 2026-03-20T11:46:50.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.204+0000 7fdbacf23900 20 run: stack=0x5608c1237310 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.204+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1205620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.204+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1205620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.204+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15.5 2026-03-20T11:46:50.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.204+0000 7fdbacf23900 20 run: stack=0x5608c11d6650 is io blocked 2026-03-20T11:46:50.202 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.205+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c126c0b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.202 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.205+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c126c0b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.202 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.205+0000 7fdbacf23900 15 stack 0x5608c11d3250 end 2026-03-20T11:46:50.202 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.205+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.202 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.205+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is done 2026-03-20T11:46:50.202 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.205+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c12082f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.202 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.205+0000 7fdbacf23900 20 run: stack=0x5608c0d5b650 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.203 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c120c840:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c120c840:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 15 stack 0x5608c11c55c0 end 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 run: stack=0x5608c11c55c0 is done 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c12082f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 run: stack=0x5608c0d5b650 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c12699f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c12699f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 15 stack 0x5608c11c5dd0 end 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is done 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c12325c0:op=0x5608c1240060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c12325c0:op=0x5608c1240060:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 15 stack 0x5608c12325c0 end 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 run: stack=0x5608c12325c0 is done 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11d3c90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c11d3c90:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 15 stack 0x5608c10a18e0 end 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is done 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c1240ba0:op=0x5608c123e5c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c1240ba0:op=0x5608c123e5c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 15 stack 0x5608c1240ba0 end 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 run: stack=0x5608c1240ba0 is done 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c12082f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c1244080:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c1244080:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 15 stack 0x5608c0d5f270 end 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 run: stack=0x5608c0d5f270 is done 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c12082f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c12082f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c12082f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c12082f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 run: stack=0x5608c0d5b650 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c11bee50:op=0x5608c123a4c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c11bee50:op=0x5608c123a4c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 15 stack 0x5608c11bee50 end 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 run: stack=0x5608c11bee50 is done 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c12082f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 run: stack=0x5608c0d5b650 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c1200330:op=0x5608c123cf10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c1200330:op=0x5608c123cf10:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 15 stack 0x5608c1200330 end 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 run: stack=0x5608c1200330 is done 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c12082f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 run: stack=0x5608c0d5b650 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c123b620:op=0x5608c123ddc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c123b620:op=0x5608c123ddc0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.204 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 15 stack 0x5608c123b620 end 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 run: stack=0x5608c123b620 is done 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c12082f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 run: stack=0x5608c0d5b650 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c1201890:op=0x5608c126dd20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c1201890:op=0x5608c126dd20:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 15 stack 0x5608c1201890 end 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 run: stack=0x5608c1201890 is done 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c12082f0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c0d5b650:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 trim: trimmed bucket instance dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 15 stack 0x5608c0d5b650 end 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 run: stack=0x5608c0d5b650 is done 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c1240ba0:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 4 starting trim on bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1205620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1205620:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 15 stack 0x5608c11d6650 end 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 run: stack=0x5608c11d6650 is done 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c1240ba0:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c1237310:op=0x5608c123c1d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c1240ba0:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 enqueued request req=0x5608c126b170 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 req: 0x5608c126b170 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 run: stack=0x5608c1240ba0 is io blocked 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c1237310:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 trim: trimmed bucket instance dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 15 stack 0x5608c1237310 end 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 run: stack=0x5608c1237310 is done 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 4 starting trim on bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1236030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1236030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdb6b7fe640 20 dequeued request req=0x5608c126b170 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdb6b7fe640 20 RGWWQ: empty 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 enqueued request req=0x5608c1203b30 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 req: 0x5608c1203b30 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdb697fa640 20 dequeued request req=0x5608c1203b30 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdb697fa640 20 RGWWQ: empty 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdb6b7fe640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 : miss 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdb697fa640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 : miss 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdb6b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdb697fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.206+0000 7fdbacf23900 20 run: stack=0x5608c11d6650 is io blocked 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb6b7fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb6b7fe640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 info.flags=0x17 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb6b7fe640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 to cache LRU end 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb6b7fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb6b7fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb6b7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb6b7fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-39 : miss 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb6b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb697fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb697fa640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 info.flags=0x17 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb697fa640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 to cache LRU end 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb697fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb697fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb697fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb697fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-8 : miss 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb697fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb6b7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb6b7fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-39 info.flags=0x0 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb6b7fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-39 to cache LRU end 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb6b7fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-39 : miss 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb6b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb6b7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb6b7fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-39 info.flags=0x0 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb6b7fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-39 to cache LRU end 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb6b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding source pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb697fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb697fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-8 info.flags=0x0 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb6b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding dest pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb697fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-8 to cache LRU end 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb697fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-8 : miss 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb697fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb6b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb6b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdb6b7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdbacf23900 20 cr:s=0x5608c1240ba0:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdbacf23900 20 cr:s=0x5608c1240ba0:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdbacf23900 20 cr:s=0x5608c1240ba0:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdbacf23900 20 cr:s=0x5608c1240ba0:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.207+0000 7fdbacf23900 20 cr:s=0x5608c1240ba0:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdb697fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdb697fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-8 info.flags=0x0 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdb697fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-8 to cache LRU end 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdb697fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding source pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdb697fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding dest pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdb697fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdb697fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdb697fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c0d6e300:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 cr:s=0x5608c1240ba0:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 run: stack=0x5608c1240ba0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1236030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c12082f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1236030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c12082f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 3d71c91c4b99ac768c93598453d67f264eac867f6e319313152d30f7dd38af2e 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:3d71c91c4b99ac768c93598453d67f264eac867f6e319313152d30f7dd38af2e 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = ac146fd2fd957ad51d6c056b54b6d63b5367ca9b76954e8b4fb231f1acfe7ec7 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=ac146fd2fd957ad51d6c056b54b6d63b5367ca9b76954e8b4fb231f1acfe7ec7 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=34, curl_handle=0x5608c0d5bb20 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 run: stack=0x5608c0d5f270 is io blocked 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdb067fc640 20 link_request req_data=0x5608c11cbe70 req_data->id=34, curl_handle=0x5608c0d5bb20 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1236030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1236030:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 cr:s=0x5608c12325c0:op=0x5608c1268ab0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 run: stack=0x5608c11d6650 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 cr:s=0x5608c12325c0:op=0x5608c126aad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 cr:s=0x5608c12325c0:op=0x5608c126aad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.209 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.209 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.209 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.209 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.209 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.209 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.209 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.209 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.209 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 0503df7af5150d494b3899989eafff628fd4248813431d4dbcd5c12901e0a8cb 2026-03-20T11:46:50.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.209 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.209 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.209 INFO:teuthology.orchestra.run.vm02.stderr:0503df7af5150d494b3899989eafff628fd4248813431d4dbcd5c12901e0a8cb 2026-03-20T11:46:50.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 3a62b881dd405d635c0b23447aefbf543ae2d44a7ad0edd13a4fb18da1a875a7 2026-03-20T11:46:50.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=3a62b881dd405d635c0b23447aefbf543ae2d44a7ad0edd13a4fb18da1a875a7 2026-03-20T11:46:50.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=35, curl_handle=0x5608c11cd8e0 2026-03-20T11:46:50.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdbacf23900 20 run: stack=0x5608c12325c0 is io blocked 2026-03-20T11:46:50.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.208+0000 7fdb067fc640 20 link_request req_data=0x5608c10a1a20 req_data->id=35, curl_handle=0x5608c11cd8e0 2026-03-20T11:46:50.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1240e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1240e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:50.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c0d614d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdbacf23900 15 stack 0x5608c0d6dec0 end 2026-03-20T11:46:50.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is done 2026-03-20T11:46:50.229 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdbacf23900 20 collect(): s=0x5608c11c4c90 stack=0x5608c0d6dec0 is complete 2026-03-20T11:46:50.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdbacf23900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:50.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdbacf23900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:50.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:50.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdbacf23900 15 stack 0x5608c11c4c90 end 2026-03-20T11:46:50.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdbacf23900 20 stack->operate() returned ret=-22 2026-03-20T11:46:50.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is done 2026-03-20T11:46:50.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdbacf23900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:50.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdbacf23900 4 starting trim on bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:50.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdbacf23900 20 enqueued request req=0x5608c11c67c0 2026-03-20T11:46:50.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdbacf23900 20 req: 0x5608c11c67c0 2026-03-20T11:46:50.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is io blocked 2026-03-20T11:46:50.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdb68ff9640 20 dequeued request req=0x5608c11c67c0 2026-03-20T11:46:50.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdb68ff9640 20 RGWWQ: empty 2026-03-20T11:46:50.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdb68ff9640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 : miss 2026-03-20T11:46:50.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.232+0000 7fdb68ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.233+0000 7fdb68ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:50.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.233+0000 7fdb68ff9640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 info.flags=0x17 2026-03-20T11:46:50.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.233+0000 7fdb68ff9640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 to cache LRU end 2026-03-20T11:46:50.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.233+0000 7fdb68ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.233+0000 7fdb68ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.233+0000 7fdb68ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:50.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.233+0000 7fdb68ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-16 : miss 2026-03-20T11:46:50.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.233+0000 7fdb68ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.233+0000 7fdb68ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.233+0000 7fdb68ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-16 info.flags=0x0 2026-03-20T11:46:50.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.233+0000 7fdb68ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-16 to cache LRU end 2026-03-20T11:46:50.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.233+0000 7fdb68ff9640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-16 : miss 2026-03-20T11:46:50.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.233+0000 7fdb68ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.233+0000 7fdb68ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.233+0000 7fdb68ff9640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-16 info.flags=0x0 2026-03-20T11:46:50.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.233+0000 7fdb68ff9640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-16 to cache LRU end 2026-03-20T11:46:50.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.233+0000 7fdb68ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding source pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.233+0000 7fdb68ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding dest pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.233+0000 7fdb68ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.233+0000 7fdb68ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.233+0000 7fdb68ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:50.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.233+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.233+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.233+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.233+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.233+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.234+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c0d614d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.234+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.234+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.234+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1238e60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.234+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1238e60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.234+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.234+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.234+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.234+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.234+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.234+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.234+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 06e4409fc93e9e3d50246721e1818601ebfb948a965306fed2f1653770993f71 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.234+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:06e4409fc93e9e3d50246721e1818601ebfb948a965306fed2f1653770993f71 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.234+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.234+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.234+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.234+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.234+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 2dfd65f32f1ac101a6031aac7e16ff4123d6c91edbbaae1ccd99a88f1856cc30 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.234+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2dfd65f32f1ac101a6031aac7e16ff4123d6c91edbbaae1ccd99a88f1856cc30 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.234+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.234+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.234+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.234+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=36, curl_handle=0x5608c11c9480 2026-03-20T11:46:50.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.234+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is io blocked 2026-03-20T11:46:50.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.234+0000 7fdb067fc640 20 link_request req_data=0x5608c1208920 req_data->id=36, curl_handle=0x5608c11c9480 2026-03-20T11:46:50.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c0d6b640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c0d6b640:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:50.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c12042e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdbacf23900 15 stack 0x5608c120c460 end 2026-03-20T11:46:50.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdbacf23900 20 run: stack=0x5608c120c460 is done 2026-03-20T11:46:50.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdbacf23900 20 collect(): s=0x5608c11c5f10 stack=0x5608c120c460 is complete 2026-03-20T11:46:50.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdbacf23900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:46:50.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdbacf23900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:50.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:50.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdbacf23900 15 stack 0x5608c11c5f10 end 2026-03-20T11:46:50.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdbacf23900 20 stack->operate() returned ret=-22 2026-03-20T11:46:50.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is done 2026-03-20T11:46:50.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdbacf23900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:50.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdbacf23900 4 starting trim on bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:50.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdbacf23900 20 enqueued request req=0x5608c11d6400 2026-03-20T11:46:50.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdbacf23900 20 req: 0x5608c11d6400 2026-03-20T11:46:50.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is io blocked 2026-03-20T11:46:50.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdb63fff640 20 dequeued request req=0x5608c11d6400 2026-03-20T11:46:50.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdb63fff640 20 RGWWQ: empty 2026-03-20T11:46:50.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdb63fff640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 : miss 2026-03-20T11:46:50.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdb63fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdb63fff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=314 2026-03-20T11:46:50.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdb63fff640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 info.flags=0x17 2026-03-20T11:46:50.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdb63fff640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 to cache LRU end 2026-03-20T11:46:50.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdb63fff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdb63fff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdb63fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:50.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdb63fff640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-43 : miss 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdb63fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdb63fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdb63fff640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-43 info.flags=0x0 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdb63fff640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-43 to cache LRU end 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdb63fff640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-43 : miss 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.242+0000 7fdb63fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdb63fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdb63fff640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-43 info.flags=0x0 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdb63fff640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-43 to cache LRU end 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdb63fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding source pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdb63fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding dest pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdb63fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdb63fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdb63fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c12042e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c0d6b640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c0d6b640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.241 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.242 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = daeb729701203b27af79f411ccc867dc6a398b086dab6fd165a8d5687a58b61f 2026-03-20T11:46:50.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.242 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.242 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.242 INFO:teuthology.orchestra.run.vm02.stderr:daeb729701203b27af79f411ccc867dc6a398b086dab6fd165a8d5687a58b61f 2026-03-20T11:46:50.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 5363690c7a1c29ebfbaa02beabb747c0b75f0235e8e568e6088348017da055bc 2026-03-20T11:46:50.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5363690c7a1c29ebfbaa02beabb747c0b75f0235e8e568e6088348017da055bc 2026-03-20T11:46:50.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=37, curl_handle=0x5608c0d60ac0 2026-03-20T11:46:50.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdbacf23900 20 run: stack=0x5608c120c460 is io blocked 2026-03-20T11:46:50.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.243+0000 7fdb067fc640 20 link_request req_data=0x5608c11d13d0 req_data->id=37, curl_handle=0x5608c0d60ac0 2026-03-20T11:46:50.242 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:50.243+0000 7fd0582fe640 -1 req 12584368050781554749 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:50.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c12082f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c12082f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c12325c0:op=0x5608c126aad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c12082f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c12325c0:op=0x5608c126aad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c12082f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c12325c0:op=0x5608c126aad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c0d6e300:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 15 stack 0x5608c0d5f270 end 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 run: stack=0x5608c0d5f270 is done 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c12325c0:op=0x5608c126aad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c1240ba0:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 collect(): s=0x5608c1240ba0 stack=0x5608c0d5f270 is complete 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 10 trimming bilogs for bucket=:dipepu-39[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44]) markers=[,,,,,,,,00000000001.10.6,,], shards=11 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c12325c0:op=0x5608c1268ab0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 15 stack 0x5608c12325c0 end 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 run: stack=0x5608c12325c0 is done 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c1240ba0:op=0x5608c0d6e300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 10 trimming bilog shard 8 of :dipepu-39[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44]) at marker 00000000001.10.6 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 run: stack=0x5608c1240ba0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 collect(): s=0x5608c11d6650 stack=0x5608c12325c0 is complete 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 10 trimming bilogs for bucket=:dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) markers=[,00000000002.12.6,,00000000001.5.6,,00000000001.4.6,00000000002.12.6,00000000001.8.6,00000000001.14.6,00000000001.9.6,], shards=11 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c1205620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1268ab0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 10 trimming bilog shard 1 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000002.12.6 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 10 trimming bilog shard 3 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.5.6 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 10 trimming bilog shard 5 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.4.6 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 10 trimming bilog shard 6 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000002.12.6 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 10 trimming bilog shard 7 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.8.6 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 10 trimming bilog shard 8 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.14.6 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 10 trimming bilog shard 9 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.9.6 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 run: stack=0x5608c11d6650 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c1205620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44.8 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 run: stack=0x5608c0d5f270 is io blocked 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c12325c0:op=0x5608c120c990:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123cf10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c1240060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c11d1c40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c126bd10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c123b4e0:op=0x5608c123a160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c123c3f0:op=0x5608c126ca40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c12325c0:op=0x5608c120c990:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.1 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 run: stack=0x5608c12325c0 is io blocked 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123cf10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.3 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is io blocked 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c1240060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.5 2026-03-20T11:46:50.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is io blocked 2026-03-20T11:46:50.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c11d1c40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.6 2026-03-20T11:46:50.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 run: stack=0x5608c11c55c0 is io blocked 2026-03-20T11:46:50.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c126bd10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.7 2026-03-20T11:46:50.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is io blocked 2026-03-20T11:46:50.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c123b4e0:op=0x5608c123a160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.8 2026-03-20T11:46:50.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 run: stack=0x5608c123b4e0 is io blocked 2026-03-20T11:46:50.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 cr:s=0x5608c123c3f0:op=0x5608c126ca40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.9 2026-03-20T11:46:50.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.251+0000 7fdbacf23900 20 run: stack=0x5608c123c3f0 is io blocked 2026-03-20T11:46:50.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c12325c0:op=0x5608c120c990:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c12325c0:op=0x5608c120c990:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 15 stack 0x5608c12325c0 end 2026-03-20T11:46:50.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 run: stack=0x5608c12325c0 is done 2026-03-20T11:46:50.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1268ab0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 run: stack=0x5608c11d6650 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c123c3f0:op=0x5608c126ca40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c123c3f0:op=0x5608c126ca40:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 15 stack 0x5608c123c3f0 end 2026-03-20T11:46:50.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 run: stack=0x5608c123c3f0 is done 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1268ab0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 run: stack=0x5608c11d6650 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123cf10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123cf10:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 15 stack 0x5608c10a18e0 end 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is done 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c1240060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c1240060:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 15 stack 0x5608c11c5dd0 end 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is done 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c126bd10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c126bd10:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 15 stack 0x5608c11d3250 end 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is done 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c1205620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c1205620:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 15 stack 0x5608c0d5f270 end 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 run: stack=0x5608c0d5f270 is done 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c11d1c40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c11d1c40:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 15 stack 0x5608c11c55c0 end 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 run: stack=0x5608c11c55c0 is done 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1268ab0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c1240ba0:op=0x5608c0d6e300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1268ab0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c1240ba0:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 trim: trimmed bucket instance dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 15 stack 0x5608c1240ba0 end 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 run: stack=0x5608c1240ba0 is done 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1268ab0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1268ab0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 run: stack=0x5608c11d6650 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 4 starting trim on bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c1242e20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c1242e20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 enqueued request req=0x5608c12367c0 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 req: 0x5608c12367c0 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is io blocked 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdb637fe640 20 dequeued request req=0x5608c12367c0 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdb637fe640 20 RGWWQ: empty 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdb637fe640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 : miss 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c123b4e0:op=0x5608c123a160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c123b4e0:op=0x5608c123a160:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 15 stack 0x5608c123b4e0 end 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 run: stack=0x5608c123b4e0 is done 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c1268ab0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c11d6650:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 trim: trimmed bucket instance dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 15 stack 0x5608c11d6650 end 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdb637fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 run: stack=0x5608c11d6650 is done 2026-03-20T11:46:50.252 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.252 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.252 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.252 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 4 starting trim on bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:50.252 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1233f20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.252 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1233f20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.252 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 enqueued request req=0x5608c11d52c0 2026-03-20T11:46:50.252 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.252 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 req: 0x5608c11d52c0 2026-03-20T11:46:50.252 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is io blocked 2026-03-20T11:46:50.252 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdb62ffd640 20 dequeued request req=0x5608c11d52c0 2026-03-20T11:46:50.252 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdb62ffd640 20 RGWWQ: empty 2026-03-20T11:46:50.252 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdb62ffd640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 : miss 2026-03-20T11:46:50.252 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.253+0000 7fdb62ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.252 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb637fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:50.252 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb637fe640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 info.flags=0x17 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb637fe640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 to cache LRU end 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb637fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb637fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb637fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb637fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-10 : miss 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb637fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb62ffd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb62ffd640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 info.flags=0x17 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb62ffd640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 to cache LRU end 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb62ffd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb62ffd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb62ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb62ffd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-3 : miss 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb62ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb637fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb637fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-10 info.flags=0x0 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb637fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-10 to cache LRU end 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb637fe640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-10 : miss 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb637fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb637fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb637fe640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-10 info.flags=0x0 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb637fe640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-10 to cache LRU end 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb637fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding source pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb637fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding dest pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb637fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb637fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb637fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb62ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb62ffd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-3 info.flags=0x0 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb62ffd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-3 to cache LRU end 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb62ffd640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-3 : miss 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdb62ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c1242e20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c1242e20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c1242e20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c1242e20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c0d6e300:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c1268ab0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c1268ab0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.254+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdb62ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdb62ffd640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-3 info.flags=0x0 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdb62ffd640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-3 to cache LRU end 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdb62ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding source pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdb62ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding dest pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdb62ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdb62ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.253 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdb62ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 4d64c44486bf966c42a04bfeded047cdb888df961a0c9de0be877aa03d8f656f 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:4d64c44486bf966c42a04bfeded047cdb888df961a0c9de0be877aa03d8f656f 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = cae679735d0a00ec260ae1fe20fa21f416c2aeb7dd58959294057d26997413b2 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=cae679735d0a00ec260ae1fe20fa21f416c2aeb7dd58959294057d26997413b2 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=38, curl_handle=0x5608c11cd8e0 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 20 run: stack=0x5608c0d5f270 is io blocked 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdb067fc640 20 link_request req_data=0x5608c10a1a20 req_data->id=38, curl_handle=0x5608c11cd8e0 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1233f20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1233f20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1233f20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1233f20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c12377e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c12082f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c12082f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 916918bfdbfc4d7aa3903692d188cb432c17ac86797bacfd0dc7ebe88c621361 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:916918bfdbfc4d7aa3903692d188cb432c17ac86797bacfd0dc7ebe88c621361 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = e491a9b6aad93a59c8433f887fc542e7f8830d0a5321ddf02eabf33f301d295f 2026-03-20T11:46:50.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=e491a9b6aad93a59c8433f887fc542e7f8830d0a5321ddf02eabf33f301d295f 2026-03-20T11:46:50.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=39, curl_handle=0x5608c0d5bb20 2026-03-20T11:46:50.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is io blocked 2026-03-20T11:46:50.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.255+0000 7fdb067fc640 20 link_request req_data=0x5608c11cbe70 req_data->id=39, curl_handle=0x5608c0d5bb20 2026-03-20T11:46:50.255 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:50.255+0000 7fd078b3f640 -1 req 2086741025241038954 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:50.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.277+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1238e60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.277+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1238e60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.277+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1238e60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.277+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1238e60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.277+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c0d614d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.277+0000 7fdbacf23900 15 stack 0x5608c0d6dec0 end 2026-03-20T11:46:50.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.277+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is done 2026-03-20T11:46:50.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.277+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.277+0000 7fdbacf23900 20 collect(): s=0x5608c11c4c90 stack=0x5608c0d6dec0 is complete 2026-03-20T11:46:50.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.277+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.277+0000 7fdbacf23900 10 trimming bilogs for bucket=:dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) markers=[,,,,,00000000003.17.10,,,00000000031.93.9,,], shards=11 2026-03-20T11:46:50.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.277+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c12427b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.277+0000 7fdbacf23900 10 trimming bilog shard 5 of :dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) at marker 00000000003.17.10 2026-03-20T11:46:50.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.277+0000 7fdbacf23900 10 trimming bilog shard 8 of :dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) at marker 00000000031.93.9 2026-03-20T11:46:50.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.277+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.277+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1285810:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.277+0000 7fdbacf23900 20 cr:s=0x5608c123c3f0:op=0x5608c1205620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.277+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1285810:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.277+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21.5 2026-03-20T11:46:50.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.277+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is io blocked 2026-03-20T11:46:50.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.277+0000 7fdbacf23900 20 cr:s=0x5608c123c3f0:op=0x5608c1205620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.277+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21.8 2026-03-20T11:46:50.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.277+0000 7fdbacf23900 20 run: stack=0x5608c123c3f0 is io blocked 2026-03-20T11:46:50.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdbacf23900 20 cr:s=0x5608c123c3f0:op=0x5608c1205620:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdbacf23900 20 cr:s=0x5608c123c3f0:op=0x5608c1205620:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdbacf23900 15 stack 0x5608c123c3f0 end 2026-03-20T11:46:50.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdbacf23900 20 run: stack=0x5608c123c3f0 is done 2026-03-20T11:46:50.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c12427b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1285810:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1285810:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdbacf23900 15 stack 0x5608c0d6dec0 end 2026-03-20T11:46:50.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is done 2026-03-20T11:46:50.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c12427b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdbacf23900 20 trim: trimmed bucket instance dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:50.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdbacf23900 15 stack 0x5608c11c4c90 end 2026-03-20T11:46:50.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is done 2026-03-20T11:46:50.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdbacf23900 4 starting trim on bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c1233f20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c1233f20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdbacf23900 20 enqueued request req=0x5608c11c67c0 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdbacf23900 20 req: 0x5608c11c67c0 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is io blocked 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdb627fc640 20 dequeued request req=0x5608c11c67c0 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdb627fc640 20 RGWWQ: empty 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdb627fc640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 : miss 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.279+0000 7fdb627fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.280+0000 7fdb627fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.280+0000 7fdb627fc640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 info.flags=0x17 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.280+0000 7fdb627fc640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 to cache LRU end 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.280+0000 7fdb627fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.280+0000 7fdb627fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.280+0000 7fdb627fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.280+0000 7fdb627fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-9 : miss 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.280+0000 7fdb627fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.280+0000 7fdb627fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.280+0000 7fdb627fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-9 info.flags=0x0 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.280+0000 7fdb627fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-9 to cache LRU end 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.280+0000 7fdb627fc640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-9 : miss 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.280+0000 7fdb627fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.280+0000 7fdb627fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.280+0000 7fdb627fc640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-9 info.flags=0x0 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.280+0000 7fdb627fc640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-9 to cache LRU end 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.280+0000 7fdb627fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding source pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.280+0000 7fdb627fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding dest pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.280+0000 7fdb627fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.280+0000 7fdb627fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.280+0000 7fdb627fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.280+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c1233f20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.280+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c1233f20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.281+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c1233f20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.281+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c1233f20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.281+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.281+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c0d614d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.281+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.281+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.281+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1238e60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.281+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1238e60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.281+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.281+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.281+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.281+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.281+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.281+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.281+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 5f2bbb3ca1ebbeca8c133f015ce6c52dedf2b848f904bda0a78065b478476701 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.281+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:5f2bbb3ca1ebbeca8c133f015ce6c52dedf2b848f904bda0a78065b478476701 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.281+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.281+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.281+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.281+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.281+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 16fe92ae328daf768ae2ba4b13e655fe3f0dc92706f206f88c57ed4402d36c2f 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.281+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=16fe92ae328daf768ae2ba4b13e655fe3f0dc92706f206f88c57ed4402d36c2f 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.281+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.281+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.281+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.281+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=40, curl_handle=0x5608c11c9480 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.281+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is io blocked 2026-03-20T11:46:50.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.281+0000 7fdb067fc640 20 link_request req_data=0x5608c1208920 req_data->id=40, curl_handle=0x5608c11c9480 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c0d6b640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c0d6b640:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c12042e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdbacf23900 15 stack 0x5608c120c460 end 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdbacf23900 20 run: stack=0x5608c120c460 is done 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdbacf23900 20 collect(): s=0x5608c11c5f10 stack=0x5608c120c460 is complete 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdbacf23900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:1for generation:0 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdbacf23900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdbacf23900 15 stack 0x5608c11c5f10 end 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdbacf23900 20 stack->operate() returned ret=-22 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is done 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdbacf23900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdbacf23900 4 starting trim on bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1233f20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1233f20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdbacf23900 20 enqueued request req=0x5608c11d6400 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdbacf23900 20 req: 0x5608c11d6400 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is io blocked 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdb61ffb640 20 dequeued request req=0x5608c11d6400 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdb61ffb640 20 RGWWQ: empty 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdb61ffb640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 : miss 2026-03-20T11:46:50.282 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdb61ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdb61ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdb61ffb640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 info.flags=0x17 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdb61ffb640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 to cache LRU end 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdb61ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdb61ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdb61ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdb61ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-38 : miss 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdb61ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdb61ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdb61ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-38 info.flags=0x0 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdb61ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-38 to cache LRU end 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdb61ffb640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-38 : miss 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.285+0000 7fdb61ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdb61ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdb61ffb640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-38 info.flags=0x0 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdb61ffb640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-38 to cache LRU end 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdb61ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding source pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdb61ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding dest pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdb61ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdb61ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdb61ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1233f20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1233f20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1233f20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1233f20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c12042e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c0d6b640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c0d6b640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.283 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.284 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.284 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.284 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.284 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.284 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.284 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.284 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.284 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 02975ba013c77077935ffa5569f39a8b18ed1d08e14def79279c1d00436348c0 2026-03-20T11:46:50.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.284 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.284 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.284 INFO:teuthology.orchestra.run.vm02.stderr:02975ba013c77077935ffa5569f39a8b18ed1d08e14def79279c1d00436348c0 2026-03-20T11:46:50.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 1ae1f24ff81ecdab7d0b0558d3f833f96557dc6cfc3ac79d569958724be6fb4b 2026-03-20T11:46:50.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=1ae1f24ff81ecdab7d0b0558d3f833f96557dc6cfc3ac79d569958724be6fb4b 2026-03-20T11:46:50.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=41, curl_handle=0x5608c0d60ac0 2026-03-20T11:46:50.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdbacf23900 20 run: stack=0x5608c120c460 is io blocked 2026-03-20T11:46:50.284 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.286+0000 7fdb067fc640 20 link_request req_data=0x5608c11d13d0 req_data->id=41, curl_handle=0x5608c0d60ac0 2026-03-20T11:46:50.294 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c1268ab0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c1268ab0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c12082f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c12082f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c1268ab0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c12377e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 15 stack 0x5608c10a18e0 end 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is done 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c1268ab0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 collect(): s=0x5608c11d3250 stack=0x5608c10a18e0 is complete 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 15 stack 0x5608c11d3250 end 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 stack->operate() returned ret=-22 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is done 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c0d6e300:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 15 stack 0x5608c0d5f270 end 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 run: stack=0x5608c0d5f270 is done 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 collect(): s=0x5608c11c5dd0 stack=0x5608c0d5f270 is complete 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 10 trimming bilogs for bucket=:dipepu-10[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15]) markers=[,00000000002.9.6,,,,,,,,,], shards=11 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 4 starting trim on bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c0d6e300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 10 trimming bilog shard 1 of :dipepu-10[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15]) at marker 00000000002.9.6 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c1231090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 enqueued request req=0x5608c126bb00 2026-03-20T11:46:50.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 req: 0x5608c126bb00 2026-03-20T11:46:50.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is io blocked 2026-03-20T11:46:50.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.298+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c1231090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.297+0000 7fdb617fa640 20 dequeued request req=0x5608c126bb00 2026-03-20T11:46:50.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.298+0000 7fdb617fa640 20 RGWWQ: empty 2026-03-20T11:46:50.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.298+0000 7fdb617fa640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 : miss 2026-03-20T11:46:50.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.298+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15.1 2026-03-20T11:46:50.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.298+0000 7fdb617fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.298+0000 7fdbacf23900 20 run: stack=0x5608c0d5f270 is io blocked 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.298+0000 7fdb617fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.298+0000 7fdb617fa640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 info.flags=0x17 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.298+0000 7fdb617fa640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 to cache LRU end 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.298+0000 7fdb617fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.298+0000 7fdb617fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.298+0000 7fdb617fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.298+0000 7fdb617fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-15 : miss 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.298+0000 7fdb617fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdb617fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdb617fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-15 info.flags=0x0 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdb617fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-15 to cache LRU end 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdb617fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-15 : miss 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdb617fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c1231090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c1231090:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 15 stack 0x5608c0d5f270 end 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 run: stack=0x5608c0d5f270 is done 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c0d6e300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdb617fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdb617fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-15 info.flags=0x0 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 trim: trimmed bucket instance dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 15 stack 0x5608c11c5dd0 end 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdb617fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-15 to cache LRU end 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is done 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdb617fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding source pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 4 starting trim on bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c126c9c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c126c9c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 enqueued request req=0x5608c0d5c050 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.297 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 req: 0x5608c0d5c050 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is io blocked 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdb617fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding dest pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdb617fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdb617fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdb617fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdb617fa640 20 dequeued request req=0x5608c0d5c050 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdb617fa640 20 RGWWQ: empty 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c11d5690:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdb617fa640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 : miss 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdb617fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c0d6e300:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c1268ab0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c1268ab0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 1b3f41e2120673230511140870be1d9ae3e9209a6882fe370c569f73a201dd8d 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:1b3f41e2120673230511140870be1d9ae3e9209a6882fe370c569f73a201dd8d 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.298 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 354ff1a87ffc3b895cb6b2a65605efd47a9c4493706acf01709783e3dbc6abda 2026-03-20T11:46:50.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=354ff1a87ffc3b895cb6b2a65605efd47a9c4493706acf01709783e3dbc6abda 2026-03-20T11:46:50.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.299+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=42, curl_handle=0x5608c0d5bb20 2026-03-20T11:46:50.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.300+0000 7fdb067fc640 20 link_request req_data=0x5608c11cbe70 req_data->id=42, curl_handle=0x5608c0d5bb20 2026-03-20T11:46:50.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.300+0000 7fdbacf23900 20 run: stack=0x5608c0d5f270 is io blocked 2026-03-20T11:46:50.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.300+0000 7fdb617fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:50.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.300+0000 7fdb617fa640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 info.flags=0x17 2026-03-20T11:46:50.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.300+0000 7fdb617fa640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 to cache LRU end 2026-03-20T11:46:50.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.300+0000 7fdb617fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.300+0000 7fdb617fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.300+0000 7fdb617fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:50.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.300+0000 7fdb617fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-13 : miss 2026-03-20T11:46:50.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.300+0000 7fdb617fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.300+0000 7fdb617fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.300+0000 7fdb617fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-13 info.flags=0x0 2026-03-20T11:46:50.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.300+0000 7fdb617fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-13 to cache LRU end 2026-03-20T11:46:50.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.300+0000 7fdb617fa640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-13 : miss 2026-03-20T11:46:50.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.300+0000 7fdb617fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.300+0000 7fdb617fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.300+0000 7fdb617fa640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-13 info.flags=0x0 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.300+0000 7fdb617fa640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-13 to cache LRU end 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.300+0000 7fdb617fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18): adding source pipe: {s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.300+0000 7fdb617fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18): adding dest pipe: {s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.300+0000 7fdb617fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.300+0000 7fdb617fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdb617fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c126c9c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c126c9c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c126c9c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c126c9c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c12082f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c126c100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c126c100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 9357a93adc6bd86c05c8b59878176275e19c799fa2aee95c27f2657c56ce6c72 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.300 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.301 INFO:teuthology.orchestra.run.vm02.stderr:9357a93adc6bd86c05c8b59878176275e19c799fa2aee95c27f2657c56ce6c72 2026-03-20T11:46:50.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = c61bd1ba773f53bcd495459e6b925396e4baa2636437b7c72f3a987c6f11ea68 2026-03-20T11:46:50.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c61bd1ba773f53bcd495459e6b925396e4baa2636437b7c72f3a987c6f11ea68 2026-03-20T11:46:50.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=43, curl_handle=0x5608c11cd8e0 2026-03-20T11:46:50.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdb067fc640 20 link_request req_data=0x5608c10a1a20 req_data->id=43, curl_handle=0x5608c11cd8e0 2026-03-20T11:46:50.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.301+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is io blocked 2026-03-20T11:46:50.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.323+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1238e60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.323+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1238e60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.323+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1238e60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.323+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c1238e60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.323+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c0d614d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.323+0000 7fdbacf23900 15 stack 0x5608c0d6dec0 end 2026-03-20T11:46:50.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.323+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is done 2026-03-20T11:46:50.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.323+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.323+0000 7fdbacf23900 20 collect(): s=0x5608c11c4c90 stack=0x5608c0d6dec0 is complete 2026-03-20T11:46:50.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.323+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.323+0000 7fdbacf23900 10 trimming bilogs for bucket=:dipepu-9[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14]) markers=[,00000000002.13.6,,,,,,,,,], shards=11 2026-03-20T11:46:50.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.323+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c126cf30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.323+0000 7fdbacf23900 10 trimming bilog shard 1 of :dipepu-9[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14]) at marker 00000000002.13.6 2026-03-20T11:46:50.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.323+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.323+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c12369d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.323+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c12369d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.323+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14.1 2026-03-20T11:46:50.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.323+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is io blocked 2026-03-20T11:46:50.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.324+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c12369d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.324+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c12369d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.324+0000 7fdbacf23900 15 stack 0x5608c0d6dec0 end 2026-03-20T11:46:50.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.324+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.324+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is done 2026-03-20T11:46:50.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.324+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c126cf30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.324+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.324+0000 7fdbacf23900 20 trim: trimmed bucket instance dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:50.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.324+0000 7fdbacf23900 15 stack 0x5608c11c4c90 end 2026-03-20T11:46:50.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.324+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is done 2026-03-20T11:46:50.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.324+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.324+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.324+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.324+0000 7fdbacf23900 4 starting trim on bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:50.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.324+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11d1c40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.324+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11d1c40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.324+0000 7fdbacf23900 20 enqueued request req=0x5608c11c67c0 2026-03-20T11:46:50.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.324+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.324+0000 7fdbacf23900 20 req: 0x5608c11c67c0 2026-03-20T11:46:50.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.324+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is io blocked 2026-03-20T11:46:50.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.324+0000 7fdb607f8640 20 dequeued request req=0x5608c11c67c0 2026-03-20T11:46:50.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.324+0000 7fdb607f8640 20 RGWWQ: empty 2026-03-20T11:46:50.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.325+0000 7fdb607f8640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 : miss 2026-03-20T11:46:50.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.325+0000 7fdb607f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.325+0000 7fdb607f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:50.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.325+0000 7fdb607f8640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 info.flags=0x17 2026-03-20T11:46:50.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.325+0000 7fdb607f8640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 to cache LRU end 2026-03-20T11:46:50.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.325+0000 7fdb607f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.325+0000 7fdb607f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.325+0000 7fdb607f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:50.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.325+0000 7fdb607f8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-11 : miss 2026-03-20T11:46:50.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.325+0000 7fdb607f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.325+0000 7fdb607f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.325+0000 7fdb607f8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-11 info.flags=0x0 2026-03-20T11:46:50.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.325+0000 7fdb607f8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-11 to cache LRU end 2026-03-20T11:46:50.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.325+0000 7fdb607f8640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-11 : miss 2026-03-20T11:46:50.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.325+0000 7fdb607f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdb607f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdb607f8640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-11 info.flags=0x0 2026-03-20T11:46:50.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdb607f8640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-11 to cache LRU end 2026-03-20T11:46:50.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdb607f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16): adding source pipe: {s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdb607f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16): adding dest pipe: {s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdb607f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdb607f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdb607f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11d1c40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11d1c40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11d1c40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11d1c40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c0d614d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c123c530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c123c530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 4d5708327500d4bfbec0913b5653aa2d10868172fece53614a1b1df4414b66b5 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.324 INFO:teuthology.orchestra.run.vm02.stderr:4d5708327500d4bfbec0913b5653aa2d10868172fece53614a1b1df4414b66b5 2026-03-20T11:46:50.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 6ce8146e3581a3e519269b31b5cbc2d5916dca1428fab84d4a98cd4f9ee96661 2026-03-20T11:46:50.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=6ce8146e3581a3e519269b31b5cbc2d5916dca1428fab84d4a98cd4f9ee96661 2026-03-20T11:46:50.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=44, curl_handle=0x5608c11c9480 2026-03-20T11:46:50.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is io blocked 2026-03-20T11:46:50.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.326+0000 7fdb067fc640 20 link_request req_data=0x5608c1208920 req_data->id=44, curl_handle=0x5608c11c9480 2026-03-20T11:46:50.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.329+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c0d6b640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.326 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.329+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c0d6b640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.326 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.329+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c0d6b640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.326 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.329+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c0d6b640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.326 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.329+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c12042e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.326 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.329+0000 7fdbacf23900 15 stack 0x5608c120c460 end 2026-03-20T11:46:50.326 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.329+0000 7fdbacf23900 20 run: stack=0x5608c120c460 is done 2026-03-20T11:46:50.326 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.329+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.326 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.329+0000 7fdbacf23900 20 collect(): s=0x5608c11c5f10 stack=0x5608c120c460 is complete 2026-03-20T11:46:50.326 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.329+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.326 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.329+0000 7fdbacf23900 10 trimming bilogs for bucket=:dipepu-38[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43]) markers=[,,,,,,,,00000000001.38.6,,], shards=11 2026-03-20T11:46:50.326 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.329+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c0d6b640:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.326 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.329+0000 7fdbacf23900 10 trimming bilog shard 8 of :dipepu-38[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43]) at marker 00000000001.38.6 2026-03-20T11:46:50.326 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.329+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.326 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.329+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c1242e20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.326 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.329+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c1242e20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.326 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.329+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43.8 2026-03-20T11:46:50.326 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.329+0000 7fdbacf23900 20 run: stack=0x5608c120c460 is io blocked 2026-03-20T11:46:50.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c1242e20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c1242e20:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdbacf23900 15 stack 0x5608c120c460 end 2026-03-20T11:46:50.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdbacf23900 20 run: stack=0x5608c120c460 is done 2026-03-20T11:46:50.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c0d6b640:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdbacf23900 20 trim: trimmed bucket instance dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:50.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdbacf23900 15 stack 0x5608c11c5f10 end 2026-03-20T11:46:50.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is done 2026-03-20T11:46:50.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdbacf23900 4 starting trim on bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:50.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1285850:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1285850:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdbacf23900 20 enqueued request req=0x5608c11d6400 2026-03-20T11:46:50.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdbacf23900 20 req: 0x5608c11d6400 2026-03-20T11:46:50.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is io blocked 2026-03-20T11:46:50.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdb5fff7640 20 dequeued request req=0x5608c11d6400 2026-03-20T11:46:50.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdb5fff7640 20 RGWWQ: empty 2026-03-20T11:46:50.327 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdb5fff7640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 : miss 2026-03-20T11:46:50.328 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdb5fff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.328 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdb5fff7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:50.328 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdb5fff7640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 info.flags=0x17 2026-03-20T11:46:50.328 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdb5fff7640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 to cache LRU end 2026-03-20T11:46:50.328 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdb5fff7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.328 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdb5fff7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdb5fff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdb5fff7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-12 : miss 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.330+0000 7fdb5fff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdb5fff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdb5fff7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-12 info.flags=0x0 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdb5fff7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-12 to cache LRU end 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdb5fff7640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-12 : miss 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdb5fff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdb5fff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdb5fff7640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-12 info.flags=0x0 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdb5fff7640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-12 to cache LRU end 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdb5fff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17): adding source pipe: {s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdb5fff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17): adding dest pipe: {s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdb5fff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdb5fff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdb5fff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1285850:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1285850:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1285850:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1285850:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c11cc360:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c12042e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c12042e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.329 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.330 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.330 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.330 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.330 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.330 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.330 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.330 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 55dce38bd4f521b1b48a385e48148d5489b6e8f6b96caf80b340c16f397772c8 2026-03-20T11:46:50.330 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.330 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.330 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.330 INFO:teuthology.orchestra.run.vm02.stderr:55dce38bd4f521b1b48a385e48148d5489b6e8f6b96caf80b340c16f397772c8 2026-03-20T11:46:50.330 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.330 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.330 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.330 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.330 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = bc584cddeaf014aa752fe4f1cbb0787dec83573ee54afa7ab6effc983ad8a05b 2026-03-20T11:46:50.330 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=bc584cddeaf014aa752fe4f1cbb0787dec83573ee54afa7ab6effc983ad8a05b 2026-03-20T11:46:50.330 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.330 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.330 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.330 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=45, curl_handle=0x5608c0d60ac0 2026-03-20T11:46:50.330 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdb067fc640 20 link_request req_data=0x5608c11d13d0 req_data->id=45, curl_handle=0x5608c0d60ac0 2026-03-20T11:46:50.330 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.331+0000 7fdbacf23900 20 run: stack=0x5608c120c460 is io blocked 2026-03-20T11:46:50.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.342+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c1268ab0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.342+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c1268ab0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.342+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c1268ab0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.342+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c1268ab0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.342+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c0d6e300:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.342+0000 7fdbacf23900 15 stack 0x5608c0d5f270 end 2026-03-20T11:46:50.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.342+0000 7fdbacf23900 20 run: stack=0x5608c0d5f270 is done 2026-03-20T11:46:50.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.342+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.342+0000 7fdbacf23900 20 collect(): s=0x5608c11d3250 stack=0x5608c0d5f270 is complete 2026-03-20T11:46:50.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.342+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.342+0000 7fdbacf23900 10 trimming bilogs for bucket=:dipepu-15[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20]) markers=[,,,,,,,,00000000007.20.6,,], shards=11 2026-03-20T11:46:50.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.342+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c11d1c40:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.342+0000 7fdbacf23900 10 trimming bilog shard 8 of :dipepu-15[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20]) at marker 00000000007.20.6 2026-03-20T11:46:50.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.342+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.342+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c123a200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.342+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c123a200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.342+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20.8 2026-03-20T11:46:50.339 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.342+0000 7fdbacf23900 20 run: stack=0x5608c0d5f270 is io blocked 2026-03-20T11:46:50.340 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.343+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c123a200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.343+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c123a200:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.343+0000 7fdbacf23900 15 stack 0x5608c0d5f270 end 2026-03-20T11:46:50.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.343+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.343+0000 7fdbacf23900 20 run: stack=0x5608c0d5f270 is done 2026-03-20T11:46:50.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.343+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c11d1c40:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.343+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.343+0000 7fdbacf23900 20 trim: trimmed bucket instance dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:50.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.343+0000 7fdbacf23900 15 stack 0x5608c11d3250 end 2026-03-20T11:46:50.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.343+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is done 2026-03-20T11:46:50.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.343+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.343+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.343+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.343+0000 7fdbacf23900 4 starting trim on bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:50.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.343+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c123d910:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.343+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c123d910:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.343+0000 7fdbacf23900 20 enqueued request req=0x5608c126bb00 2026-03-20T11:46:50.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.343+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.343+0000 7fdbacf23900 20 req: 0x5608c126bb00 2026-03-20T11:46:50.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.343+0000 7fdb5eff5640 20 dequeued request req=0x5608c126bb00 2026-03-20T11:46:50.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.343+0000 7fdb5eff5640 20 RGWWQ: empty 2026-03-20T11:46:50.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.343+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is io blocked 2026-03-20T11:46:50.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdb5eff5640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 : miss 2026-03-20T11:46:50.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdb5eff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.341 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c126c100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdb5eff5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdb5eff5640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 info.flags=0x17 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdb5eff5640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 to cache LRU end 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdb5eff5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdb5eff5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdb5eff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdb5eff5640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-40 : miss 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdb5eff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdb5eff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdb5eff5640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-40 info.flags=0x0 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdb5eff5640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-40 to cache LRU end 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c126c100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c126c100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c126c100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c12082f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdb5eff5640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-40 : miss 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdbacf23900 15 stack 0x5608c10a18e0 end 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is done 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdb5eff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdbacf23900 20 collect(): s=0x5608c11c5dd0 stack=0x5608c10a18e0 is complete 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdbacf23900 10 trimming bilogs for bucket=:dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) markers=[,,,,,00000000003.29.10,,,00000000003.11.6,00000000011.37.6,], shards=11 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c12427b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdbacf23900 10 trimming bilog shard 5 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000003.29.10 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdbacf23900 10 trimming bilog shard 8 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000003.11.6 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdbacf23900 10 trimming bilog shard 9 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000011.37.6 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123a200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c1241ea0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdbacf23900 20 cr:s=0x5608c123c3f0:op=0x5608c123e550:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123a200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.5 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is io blocked 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c1241ea0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.8 2026-03-20T11:46:50.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdbacf23900 20 run: stack=0x5608c0d5f270 is io blocked 2026-03-20T11:46:50.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdbacf23900 20 cr:s=0x5608c123c3f0:op=0x5608c123e550:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.9 2026-03-20T11:46:50.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.344+0000 7fdbacf23900 20 run: stack=0x5608c123c3f0 is io blocked 2026-03-20T11:46:50.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdb5eff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdb5eff5640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-40 info.flags=0x0 2026-03-20T11:46:50.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdb5eff5640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-40 to cache LRU end 2026-03-20T11:46:50.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdb5eff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1): adding source pipe: {s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdb5eff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1): adding dest pipe: {s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdb5eff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdb5eff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdb5eff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:50.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c123d910:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c123d910:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c123d910:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c123d910:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 20 cr:s=0x5608c12325c0:op=0x5608c0d6e300:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 20 cr:s=0x5608c12325c0:op=0x5608c1268ab0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 20 cr:s=0x5608c12325c0:op=0x5608c1268ab0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 637e79fc86e107551bf183273ab1f36340b008073f0daf341742359bb189ec42 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:637e79fc86e107551bf183273ab1f36340b008073f0daf341742359bb189ec42 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 717ba4b6bce674b17ef650ed8fc0c84d9f7c655953ba4b29e5138eaa3ba779a5 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=717ba4b6bce674b17ef650ed8fc0c84d9f7c655953ba4b29e5138eaa3ba779a5 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=46, curl_handle=0x5608c11cd8e0 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdbacf23900 20 run: stack=0x5608c12325c0 is io blocked 2026-03-20T11:46:50.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.345+0000 7fdb067fc640 20 link_request req_data=0x5608c10a1a20 req_data->id=46, curl_handle=0x5608c11cd8e0 2026-03-20T11:46:50.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123a200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c123a200:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 15 stack 0x5608c10a18e0 end 2026-03-20T11:46:50.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is done 2026-03-20T11:46:50.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c12427b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 20 cr:s=0x5608c123c3f0:op=0x5608c123e550:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 20 cr:s=0x5608c123c3f0:op=0x5608c123e550:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 15 stack 0x5608c123c3f0 end 2026-03-20T11:46:50.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 20 run: stack=0x5608c123c3f0 is done 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c1241ea0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c1241ea0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 15 stack 0x5608c0d5f270 end 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 20 run: stack=0x5608c0d5f270 is done 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c12427b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c12427b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 20 trim: trimmed bucket instance dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 15 stack 0x5608c11c5dd0 end 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is done 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 4 starting trim on bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c123d910:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c123d910:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 20 enqueued request req=0x5608c0d5c050 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 20 req: 0x5608c0d5c050 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is io blocked 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdb5f7f6640 20 dequeued request req=0x5608c0d5c050 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdb5f7f6640 20 RGWWQ: empty 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdb5f7f6640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 : miss 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.346+0000 7fdb5f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdb5f7f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdb5f7f6640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 info.flags=0x17 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdb5f7f6640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 to cache LRU end 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdb5f7f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdb5f7f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdb5f7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdb5f7f6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-18 : miss 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdb5f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdb5f7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdb5f7f6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-18 info.flags=0x0 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdb5f7f6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-18 to cache LRU end 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdb5f7f6640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-18 : miss 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdb5f7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdb5f7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdb5f7f6640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-18 info.flags=0x0 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdb5f7f6640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-18 to cache LRU end 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdb5f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23): adding source pipe: {s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdb5f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23): adding dest pipe: {s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdb5f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdb5f7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdb5f7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c123d910:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c123d910:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c123d910:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c123d910:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdbacf23900 20 cr:s=0x5608c123c3f0:op=0x5608c12082f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdbacf23900 20 cr:s=0x5608c123c3f0:op=0x5608c12057d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdbacf23900 20 cr:s=0x5608c123c3f0:op=0x5608c12057d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = e2bcf27406928f63066901f843d362204c5f345226093b31a2a85a314eb2adf6 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:e2bcf27406928f63066901f843d362204c5f345226093b31a2a85a314eb2adf6 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = edbb5a0f8f4eec020371021a7a4a65b3a5ad117030a2279c1f943de592f7ba26 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=edbb5a0f8f4eec020371021a7a4a65b3a5ad117030a2279c1f943de592f7ba26 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.347+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.348+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=47, curl_handle=0x5608c0d5bb20 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.348+0000 7fdbacf23900 20 run: stack=0x5608c123c3f0 is io blocked 2026-03-20T11:46:50.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.348+0000 7fdb067fc640 20 link_request req_data=0x5608c11cbe70 req_data->id=47, curl_handle=0x5608c0d5bb20 2026-03-20T11:46:50.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c123c530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c123c530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c123c530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c123c530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c0d614d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 15 stack 0x5608c0d6dec0 end 2026-03-20T11:46:50.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is done 2026-03-20T11:46:50.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 collect(): s=0x5608c11c4c90 stack=0x5608c0d6dec0 is complete 2026-03-20T11:46:50.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 10 trimming bilogs for bucket=:dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) markers=[00000000014.52.6,00000000008.34.6,00000000008.50.6,00000000008.32.6,00000000012.28.6,00000000010.31.6,00000000008.21.6,00000000010.27.6,00000000004.27.6,00000000008.22.6,00000000008.19.6], shards=11 2026-03-20T11:46:50.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c123e100:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 10 trimming bilog shard 0 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000014.52.6 2026-03-20T11:46:50.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 10 trimming bilog shard 1 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.34.6 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 10 trimming bilog shard 2 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.50.6 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 10 trimming bilog shard 3 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.32.6 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 10 trimming bilog shard 4 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000012.28.6 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 10 trimming bilog shard 5 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000010.31.6 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 10 trimming bilog shard 6 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.21.6 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 10 trimming bilog shard 7 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000010.27.6 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 10 trimming bilog shard 8 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000004.27.6 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 10 trimming bilog shard 9 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.22.6 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 10 trimming bilog shard 10 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.19.6 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c123a200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c11d1c40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c1269d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 cr:s=0x5608c11bee50:op=0x5608c123c530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 cr:s=0x5608c1237e00:op=0x5608c1285530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c126bcf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 cr:s=0x5608c123aa20:op=0x5608c1289c60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 cr:s=0x5608c123ef50:op=0x5608c1240060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 cr:s=0x5608c1243d60:op=0x5608c12867d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 cr:s=0x5608c1203c10:op=0x5608c1286fd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 cr:s=0x5608c123cd30:op=0x5608c12877d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c123a200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.0 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is io blocked 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c11d1c40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.1 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 run: stack=0x5608c0d5f270 is io blocked 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c1269d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.2 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is io blocked 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 cr:s=0x5608c11bee50:op=0x5608c123c530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.3 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 run: stack=0x5608c11bee50 is io blocked 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 cr:s=0x5608c1237e00:op=0x5608c1285530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.368+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.4 2026-03-20T11:46:50.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.369+0000 7fdbacf23900 20 run: stack=0x5608c1237e00 is io blocked 2026-03-20T11:46:50.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.369+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c126bcf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.369+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.5 2026-03-20T11:46:50.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.369+0000 7fdbacf23900 20 run: stack=0x5608c11c55c0 is io blocked 2026-03-20T11:46:50.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.369+0000 7fdbacf23900 20 cr:s=0x5608c123aa20:op=0x5608c1289c60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.370+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.6 2026-03-20T11:46:50.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.370+0000 7fdbacf23900 20 run: stack=0x5608c123aa20 is io blocked 2026-03-20T11:46:50.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.370+0000 7fdbacf23900 20 cr:s=0x5608c123ef50:op=0x5608c1240060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.370+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.7 2026-03-20T11:46:50.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.370+0000 7fdbacf23900 20 run: stack=0x5608c123ef50 is io blocked 2026-03-20T11:46:50.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.370+0000 7fdbacf23900 20 cr:s=0x5608c1243d60:op=0x5608c12867d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.370+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.8 2026-03-20T11:46:50.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 run: stack=0x5608c1243d60 is io blocked 2026-03-20T11:46:50.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 cr:s=0x5608c1203c10:op=0x5608c1286fd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.9 2026-03-20T11:46:50.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 run: stack=0x5608c1203c10 is io blocked 2026-03-20T11:46:50.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 cr:s=0x5608c123cd30:op=0x5608c12877d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.10 2026-03-20T11:46:50.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 run: stack=0x5608c123cd30 is io blocked 2026-03-20T11:46:50.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 cr:s=0x5608c11bee50:op=0x5608c123c530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 cr:s=0x5608c11bee50:op=0x5608c123c530:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 15 stack 0x5608c11bee50 end 2026-03-20T11:46:50.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 run: stack=0x5608c11bee50 is done 2026-03-20T11:46:50.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c123a200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c123a200:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 15 stack 0x5608c0d6dec0 end 2026-03-20T11:46:50.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is done 2026-03-20T11:46:50.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c1269d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c1269d20:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 15 stack 0x5608c10a18e0 end 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is done 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c11d1c40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c11d1c40:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 15 stack 0x5608c0d5f270 end 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 run: stack=0x5608c0d5f270 is done 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c126bcf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c126bcf0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 15 stack 0x5608c11c55c0 end 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 run: stack=0x5608c11c55c0 is done 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 cr:s=0x5608c1237e00:op=0x5608c1285530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 cr:s=0x5608c1237e00:op=0x5608c1285530:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 15 stack 0x5608c1237e00 end 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 run: stack=0x5608c1237e00 is done 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 cr:s=0x5608c123aa20:op=0x5608c1289c60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 cr:s=0x5608c123aa20:op=0x5608c1289c60:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 15 stack 0x5608c123aa20 end 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 run: stack=0x5608c123aa20 is done 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c123e100:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c123e100:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c123e100:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c123e100:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c123e100:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c123e100:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c123e100:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.371+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 cr:s=0x5608c123ef50:op=0x5608c1240060:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.369 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 cr:s=0x5608c123ef50:op=0x5608c1240060:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 15 stack 0x5608c123ef50 end 2026-03-20T11:46:50.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 run: stack=0x5608c123ef50 is done 2026-03-20T11:46:50.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c123e100:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 cr:s=0x5608c1243d60:op=0x5608c12867d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 cr:s=0x5608c1243d60:op=0x5608c12867d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 15 stack 0x5608c1243d60 end 2026-03-20T11:46:50.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 run: stack=0x5608c1243d60 is done 2026-03-20T11:46:50.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c123e100:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 cr:s=0x5608c1203c10:op=0x5608c1286fd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 cr:s=0x5608c1203c10:op=0x5608c1286fd0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 15 stack 0x5608c1203c10 end 2026-03-20T11:46:50.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 run: stack=0x5608c1203c10 is done 2026-03-20T11:46:50.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c123e100:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 cr:s=0x5608c123cd30:op=0x5608c12877d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 cr:s=0x5608c123cd30:op=0x5608c12877d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 15 stack 0x5608c123cd30 end 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 run: stack=0x5608c123cd30 is done 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c123e100:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 trim: trimmed bucket instance dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 15 stack 0x5608c11c4c90 end 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is done 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 cr:s=0x5608c123aa20:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 4 starting trim on bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 cr:s=0x5608c123aa20:op=0x5608c1231170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 cr:s=0x5608c123aa20:op=0x5608c1231170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 enqueued request req=0x5608c12688c0 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 req: 0x5608c12688c0 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdbacf23900 20 run: stack=0x5608c123aa20 is io blocked 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdb5e7f4640 20 dequeued request req=0x5608c12688c0 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdb5e7f4640 20 RGWWQ: empty 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdb5e7f4640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 : miss 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.372+0000 7fdb5e7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdb5e7f4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdb5e7f4640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 info.flags=0x17 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdb5e7f4640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 to cache LRU end 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdb5e7f4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdb5e7f4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdb5e7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdb5e7f4640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-50 : miss 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdb5e7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdb5e7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdb5e7f4640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-50 info.flags=0x0 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdb5e7f4640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-50 to cache LRU end 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdb5e7f4640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-50 : miss 2026-03-20T11:46:50.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdb5e7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdb5e7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdb5e7f4640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-50 info.flags=0x0 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdb5e7f4640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-50 to cache LRU end 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdb5e7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10): adding source pipe: {s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdb5e7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10): adding dest pipe: {s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdb5e7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdb5e7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdb5e7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 20 cr:s=0x5608c123aa20:op=0x5608c1231170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 20 cr:s=0x5608c123aa20:op=0x5608c1231170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 20 cr:s=0x5608c123aa20:op=0x5608c1231170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 20 cr:s=0x5608c123aa20:op=0x5608c1231170:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 20 cr:s=0x5608c123aa20:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c0d614d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 20 cr:s=0x5608c123aa20:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 20 run: stack=0x5608c123aa20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c1236030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c1236030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = 8f9e5e4ce69122a07e32de850af39f96d53567cee92b59e06305a30130d02d1b 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:8f9e5e4ce69122a07e32de850af39f96d53567cee92b59e06305a30130d02d1b 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = 5558960c60c79e18d5700f991407f8a66925de8e3b93152fbf502bd0031da05b 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5558960c60c79e18d5700f991407f8a66925de8e3b93152fbf502bd0031da05b 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=48, curl_handle=0x5608c11c9480 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdb067fc640 20 link_request req_data=0x5608c1208920 req_data->id=48, curl_handle=0x5608c11c9480 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.373+0000 7fdbacf23900 20 run: stack=0x5608c11c55c0 is io blocked 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c12042e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c12042e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c12042e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c12042e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c11cc360:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 15 stack 0x5608c120c460 end 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 run: stack=0x5608c120c460 is done 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 collect(): s=0x5608c11c5f10 stack=0x5608c120c460 is complete 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 10 trimming bilogs for bucket=:dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) markers=[00000000014.44.6,00000000008.52.6,00000000008.40.6,00000000008.21.6,00000000012.26.6,00000000010.26.6,00000000008.35.6,00000000010.32.6,00000000004.38.6,00000000008.22.6,00000000008.23.6], shards=11 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1285530:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 10 trimming bilog shard 0 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000014.44.6 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 10 trimming bilog shard 1 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.52.6 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 10 trimming bilog shard 2 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.40.6 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 10 trimming bilog shard 3 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.21.6 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 10 trimming bilog shard 4 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000012.26.6 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 10 trimming bilog shard 5 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000010.26.6 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 10 trimming bilog shard 6 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.35.6 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 10 trimming bilog shard 7 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000010.32.6 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 10 trimming bilog shard 8 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000004.38.6 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 10 trimming bilog shard 9 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.22.6 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 10 trimming bilog shard 10 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.23.6 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c1242e20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 cr:s=0x5608c1237e00:op=0x5608c1240420:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 cr:s=0x5608c11bee50:op=0x5608c123c530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c0d5e8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c12372a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c128a230:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c123e0d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 cr:s=0x5608c11c0450:op=0x5608c12874f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 cr:s=0x5608c1200330:op=0x5608c1287cf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 cr:s=0x5608c123ef50:op=0x5608c12884f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 cr:s=0x5608c1243d60:op=0x5608c1288cf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c1242e20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.0 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 run: stack=0x5608c120c460 is io blocked 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 cr:s=0x5608c1237e00:op=0x5608c1240420:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.1 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 run: stack=0x5608c1237e00 is io blocked 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 cr:s=0x5608c11bee50:op=0x5608c123c530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.2 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 run: stack=0x5608c11bee50 is io blocked 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.374+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c0d5e8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.375+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.3 2026-03-20T11:46:50.373 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.375+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is io blocked 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.375+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c12372a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.375+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.4 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 run: stack=0x5608c0d5f270 is io blocked 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c128a230:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.5 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is io blocked 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c123e0d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.6 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is io blocked 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 cr:s=0x5608c11c0450:op=0x5608c12874f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.7 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 run: stack=0x5608c11c0450 is io blocked 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 cr:s=0x5608c1200330:op=0x5608c1287cf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.8 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 run: stack=0x5608c1200330 is io blocked 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 cr:s=0x5608c123ef50:op=0x5608c12884f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.9 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 run: stack=0x5608c123ef50 is io blocked 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 cr:s=0x5608c1243d60:op=0x5608c1288cf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.10 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 run: stack=0x5608c1243d60 is io blocked 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c1242e20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 cr:s=0x5608c120c460:op=0x5608c1242e20:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 15 stack 0x5608c120c460 end 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 run: stack=0x5608c120c460 is done 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 cr:s=0x5608c1237e00:op=0x5608c1240420:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 cr:s=0x5608c1237e00:op=0x5608c1240420:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 15 stack 0x5608c1237e00 end 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 run: stack=0x5608c1237e00 is done 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 cr:s=0x5608c11bee50:op=0x5608c123c530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 cr:s=0x5608c11bee50:op=0x5608c123c530:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 15 stack 0x5608c11bee50 end 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 run: stack=0x5608c11bee50 is done 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1285530:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1285530:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1285530:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.374 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.376+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.377+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c0d5e8d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.377+0000 7fdbacf23900 20 cr:s=0x5608c10a18e0:op=0x5608c0d5e8d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.377+0000 7fdbacf23900 15 stack 0x5608c10a18e0 end 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.377+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.377+0000 7fdbacf23900 20 run: stack=0x5608c10a18e0 is done 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.377+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1285530:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.377+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 cr:s=0x5608c11c0450:op=0x5608c12874f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 cr:s=0x5608c11c0450:op=0x5608c12874f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 15 stack 0x5608c11c0450 end 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 run: stack=0x5608c11c0450 is done 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1285530:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c12372a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 cr:s=0x5608c0d5f270:op=0x5608c12372a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 15 stack 0x5608c0d5f270 end 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 run: stack=0x5608c0d5f270 is done 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c128a230:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c128a230:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 15 stack 0x5608c0d6dec0 end 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is done 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c123e0d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c123e0d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 15 stack 0x5608c11c4c90 end 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is done 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 cr:s=0x5608c1243d60:op=0x5608c1288cf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 cr:s=0x5608c1243d60:op=0x5608c1288cf0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 15 stack 0x5608c1243d60 end 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 run: stack=0x5608c1243d60 is done 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 cr:s=0x5608c123ef50:op=0x5608c12884f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 cr:s=0x5608c123ef50:op=0x5608c12884f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 15 stack 0x5608c123ef50 end 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 run: stack=0x5608c123ef50 is done 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1285530:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1285530:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1285530:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1285530:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1285530:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 cr:s=0x5608c1200330:op=0x5608c1287cf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 cr:s=0x5608c1200330:op=0x5608c1287cf0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 15 stack 0x5608c1200330 end 2026-03-20T11:46:50.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 run: stack=0x5608c1200330 is done 2026-03-20T11:46:50.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c1285530:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 cr:s=0x5608c11c5f10:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 trim: trimmed bucket instance dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:50.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 15 stack 0x5608c11c5f10 end 2026-03-20T11:46:50.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 run: stack=0x5608c11c5f10 is done 2026-03-20T11:46:50.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 4 starting trim on bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:50.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c123c530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c123c530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 enqueued request req=0x5608c0d5f6f0 2026-03-20T11:46:50.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 RGWWQ: 2026-03-20T11:46:50.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 req: 0x5608c0d5f6f0 2026-03-20T11:46:50.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is io blocked 2026-03-20T11:46:50.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdb5dff3640 20 dequeued request req=0x5608c0d5f6f0 2026-03-20T11:46:50.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdb5dff3640 20 RGWWQ: empty 2026-03-20T11:46:50.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdb5dff3640 10 rgw async rados processor: cache get: name=a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 : miss 2026-03-20T11:46:50.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdb5dff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdb5dff3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:50.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdb5dff3640 10 rgw async rados processor: cache put: name=a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 info.flags=0x17 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdb5dff3640 10 rgw async rados processor: adding a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 to cache LRU end 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdb5dff3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdb5dff3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdb5dff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdb5dff3640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-source-hints.dipepu-24 : miss 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.378+0000 7fdb5dff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdb5dff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdb5dff3640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-source-hints.dipepu-24 info.flags=0x0 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdb5dff3640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-source-hints.dipepu-24 to cache LRU end 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdb5dff3640 10 rgw async rados processor: cache get: name=a1.rgw.log++bucket.sync-target-hints.dipepu-24 : miss 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdb5dff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdb5dff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdb5dff3640 10 rgw async rados processor: cache put: name=a1.rgw.log++bucket.sync-target-hints.dipepu-24 info.flags=0x0 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdb5dff3640 10 rgw async rados processor: adding a1.rgw.log++bucket.sync-target-hints.dipepu-24 to cache LRU end 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdb5dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29): adding source pipe: {s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdb5dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29): adding dest pipe: {s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdb5dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdb5dff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdb5dff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c123c530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c123c530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c123c530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c123c530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c123d290:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c12042e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c12042e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 20 get_url picked endpoint=http://vm05.local:8001 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm05.local:8001 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&options=merge&rgwx-zonegroup=a&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&status=&type=bucket-index&version=2 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:50 +0000 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:host:vm05.local:8001 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114650Z 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]canonical request hash = fff7c85c342d2827523cbe3a7801052a0e033b2cd1ef37aa959312a05722dc3e 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114650Z 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:fff7c85c342d2827523cbe3a7801052a0e033b2cd1ef37aa959312a05722dc3e 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 10 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]generated signature = edd52031e898d18b7f788084f0610b12dbe79b16bc4d7844f0491239beb8ddda 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=edd52031e898d18b7f788084f0610b12dbe79b16bc4d7844f0491239beb8ddda 2026-03-20T11:46:50.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:50.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 20 http_client[GET/http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114650Z 2026-03-20T11:46:50.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 20 sending request to http://vm05.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=4b634ebf-58f5-45bc-839a-cb4b425e773a&version=2&rgwx-zonegroup=a 2026-03-20T11:46:50.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 20 register_request mgr=0x7ffcbeb5c040 req_data->id=49, curl_handle=0x5608c0d60ac0 2026-03-20T11:46:50.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is io blocked 2026-03-20T11:46:50.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.379+0000 7fdb067fc640 20 link_request req_data=0x5608c11d13d0 req_data->id=49, curl_handle=0x5608c0d60ac0 2026-03-20T11:46:50.378 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:46:50.379+0000 7fd0a839e640 -1 req 6469640969753693932 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} returned ret=-2 2026-03-20T11:46:50.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.387+0000 7fdbacf23900 20 cr:s=0x5608c12325c0:op=0x5608c1268ab0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.384 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.387+0000 7fdbacf23900 20 cr:s=0x5608c12325c0:op=0x5608c1268ab0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.384 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.387+0000 7fdbacf23900 20 cr:s=0x5608c12325c0:op=0x5608c1268ab0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.384 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.387+0000 7fdbacf23900 20 cr:s=0x5608c12325c0:op=0x5608c1268ab0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.384 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.387+0000 7fdbacf23900 20 cr:s=0x5608c12325c0:op=0x5608c0d6e300:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.384 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.387+0000 7fdbacf23900 15 stack 0x5608c12325c0 end 2026-03-20T11:46:50.384 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.387+0000 7fdbacf23900 20 run: stack=0x5608c12325c0 is done 2026-03-20T11:46:50.384 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.387+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.384 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.387+0000 7fdbacf23900 20 collect(): s=0x5608c11d3250 stack=0x5608c12325c0 is complete 2026-03-20T11:46:50.384 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.387+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.384 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.387+0000 7fdbacf23900 10 trimming bilogs for bucket=:dipepu-40[4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1]) markers=[,,,,,00000000001.103.6,,,,,], shards=11 2026-03-20T11:46:50.384 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.387+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c0d6e300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.384 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.387+0000 7fdbacf23900 10 trimming bilog shard 5 of :dipepu-40[4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1]) at marker 00000000001.103.6 2026-03-20T11:46:50.384 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.387+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.384 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.387+0000 7fdbacf23900 20 cr:s=0x5608c12325c0:op=0x5608c120c840:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.384 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.387+0000 7fdbacf23900 20 cr:s=0x5608c12325c0:op=0x5608c120c840:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.384 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.387+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1.5 2026-03-20T11:46:50.384 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.387+0000 7fdbacf23900 20 run: stack=0x5608c12325c0 is io blocked 2026-03-20T11:46:50.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.388+0000 7fdbacf23900 20 cr:s=0x5608c12325c0:op=0x5608c120c840:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.388+0000 7fdbacf23900 20 cr:s=0x5608c12325c0:op=0x5608c120c840:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.388+0000 7fdbacf23900 15 stack 0x5608c12325c0 end 2026-03-20T11:46:50.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.388+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.388+0000 7fdbacf23900 20 run: stack=0x5608c12325c0 is done 2026-03-20T11:46:50.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.388+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c0d6e300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.388+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c11bda10:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.388+0000 7fdbacf23900 20 trim: trimmed bucket instance dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:50.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.388+0000 7fdbacf23900 15 stack 0x5608c11d3250 end 2026-03-20T11:46:50.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.388+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is done 2026-03-20T11:46:50.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.388+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.388+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.390+0000 7fdbacf23900 20 cr:s=0x5608c123c3f0:op=0x5608c12057d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.390+0000 7fdbacf23900 20 cr:s=0x5608c123c3f0:op=0x5608c12057d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.390+0000 7fdbacf23900 20 cr:s=0x5608c123c3f0:op=0x5608c12057d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.390+0000 7fdbacf23900 20 cr:s=0x5608c123c3f0:op=0x5608c12057d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.390+0000 7fdbacf23900 20 cr:s=0x5608c123c3f0:op=0x5608c12082f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.390+0000 7fdbacf23900 15 stack 0x5608c123c3f0 end 2026-03-20T11:46:50.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.390+0000 7fdbacf23900 20 run: stack=0x5608c123c3f0 is done 2026-03-20T11:46:50.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.390+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.390+0000 7fdbacf23900 20 collect(): s=0x5608c11c5dd0 stack=0x5608c123c3f0 is complete 2026-03-20T11:46:50.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.390+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.390+0000 7fdbacf23900 10 trimming bilogs for bucket=:dipepu-18[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23]) markers=[,,,,,,,,00000000005.51.12,,], shards=11 2026-03-20T11:46:50.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.390+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c0d6e300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.390+0000 7fdbacf23900 10 trimming bilog shard 8 of :dipepu-18[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23]) at marker 00000000005.51.12 2026-03-20T11:46:50.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.390+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.390+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1241ea0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.390+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1241ea0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.390+0000 7fdbacf23900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23.8 2026-03-20T11:46:50.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.390+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is io blocked 2026-03-20T11:46:50.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.391+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1241ea0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:50.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.391+0000 7fdbacf23900 20 cr:s=0x5608c11d3250:op=0x5608c1241ea0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:50.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.391+0000 7fdbacf23900 15 stack 0x5608c11d3250 end 2026-03-20T11:46:50.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.391+0000 7fdbacf23900 20 stack->operate() returned ret=-61 2026-03-20T11:46:50.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.391+0000 7fdbacf23900 20 run: stack=0x5608c11d3250 is done 2026-03-20T11:46:50.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.391+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c0d6e300:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:50.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.391+0000 7fdbacf23900 20 cr:s=0x5608c11c5dd0:op=0x5608c11c37b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.391+0000 7fdbacf23900 20 trim: trimmed bucket instance dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:50.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.391+0000 7fdbacf23900 15 stack 0x5608c11c5dd0 end 2026-03-20T11:46:50.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.391+0000 7fdbacf23900 20 run: stack=0x5608c11c5dd0 is done 2026-03-20T11:46:50.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.391+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.391+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.412 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.415+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c1236030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.412 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.415+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c1236030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.412 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.415+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c1236030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.412 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.415+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c1236030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.412 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.415+0000 7fdbacf23900 20 cr:s=0x5608c11c55c0:op=0x5608c0d614d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.412 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.415+0000 7fdbacf23900 15 stack 0x5608c11c55c0 end 2026-03-20T11:46:50.412 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.415+0000 7fdbacf23900 20 run: stack=0x5608c11c55c0 is done 2026-03-20T11:46:50.412 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.415+0000 7fdbacf23900 20 cr:s=0x5608c123aa20:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.412 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.415+0000 7fdbacf23900 20 collect(): s=0x5608c123aa20 stack=0x5608c11c55c0 is complete 2026-03-20T11:46:50.412 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.415+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.412 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.415+0000 7fdbacf23900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:50.412 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.415+0000 7fdbacf23900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:50.412 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.415+0000 7fdbacf23900 20 cr:s=0x5608c123aa20:op=0x5608c11c1210:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:50.412 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.415+0000 7fdbacf23900 15 stack 0x5608c123aa20 end 2026-03-20T11:46:50.412 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.415+0000 7fdbacf23900 20 stack->operate() returned ret=-22 2026-03-20T11:46:50.412 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.415+0000 7fdbacf23900 20 run: stack=0x5608c123aa20 is done 2026-03-20T11:46:50.412 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.415+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.412 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.415+0000 7fdbacf23900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:50.412 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.415+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:50.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.420+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c12042e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:50.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.420+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c12042e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:50.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.420+0000 7fdbacf23900 20 cr:s=0x5608c0d6dec0:op=0x5608c123d290:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:50.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.420+0000 7fdbacf23900 15 stack 0x5608c0d6dec0 end 2026-03-20T11:46:50.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.420+0000 7fdbacf23900 20 run: stack=0x5608c0d6dec0 is done 2026-03-20T11:46:50.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.420+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:50.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.420+0000 7fdbacf23900 20 collect(): s=0x5608c11c4c90 stack=0x5608c0d6dec0 is complete 2026-03-20T11:46:50.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.420+0000 7fdbacf23900 10 min_generation is 0 2026-03-20T11:46:50.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.420+0000 7fdbacf23900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:50.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.420+0000 7fdbacf23900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:50.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.420+0000 7fdbacf23900 20 cr:s=0x5608c11c4c90:op=0x5608c11c24e0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:50.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.420+0000 7fdbacf23900 15 stack 0x5608c11c4c90 end 2026-03-20T11:46:50.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.420+0000 7fdbacf23900 20 stack->operate() returned ret=-22 2026-03-20T11:46:50.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.420+0000 7fdbacf23900 20 run: stack=0x5608c11c4c90 is done 2026-03-20T11:46:50.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.420+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:50.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.420+0000 7fdbacf23900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:50.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.420+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:27BucketTrimInstanceCollectCR: operate() returned r=-22 2026-03-20T11:46:50.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.420+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bc300:12BucketTrimCR: operate() 2026-03-20T11:46:50.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.420+0000 7fdbacf23900 20 writing bucket trim marker=12:4ca02412:root::dipepu-46:head 2026-03-20T11:46:50.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.420+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:50.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.420+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:50.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.420+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is io blocked 2026-03-20T11:46:50.520 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.523+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:50.520 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.523+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:50.520 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.523+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:50.520 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.523+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bf960:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:50.520 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.523+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bc300:12BucketTrimCR: operate() 2026-03-20T11:46:50.520 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.523+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c123ee10:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:50.520 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.523+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c123ee10:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:50.520 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.523+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is io blocked 2026-03-20T11:46:50.520 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.524+0000 7fdb777fe640 20 trim: bucket trim completed 2026-03-20T11:46:50.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.524+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c123ee10:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:50.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.524+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c123ee10:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:50.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.524+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c123ee10:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:50.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.524+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c123ee10:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:50.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.524+0000 7fdbacf23900 20 cr:s=0x5608c11bc0c0:op=0x5608c11bc300:12BucketTrimCR: operate() 2026-03-20T11:46:50.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.524+0000 7fdbacf23900 4 bucket index log processing completed in 0.852692783s 2026-03-20T11:46:50.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.524+0000 7fdbacf23900 15 stack 0x5608c11bc0c0 end 2026-03-20T11:46:50.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.524+0000 7fdbacf23900 20 run: stack=0x5608c11bc0c0 is done 2026-03-20T11:46:50.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.618+0000 7fdbacf23900 20 remove_watcher() i=5 2026-03-20T11:46:50.614 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.618+0000 7fdbacf23900 2 removed watcher, disabling cache 2026-03-20T11:46:50.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.619+0000 7fdbacf23900 20 remove_watcher() i=6 2026-03-20T11:46:50.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.619+0000 7fdbacf23900 20 remove_watcher() i=4 2026-03-20T11:46:50.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.619+0000 7fdbacf23900 20 remove_watcher() i=0 2026-03-20T11:46:50.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.619+0000 7fdbacf23900 20 remove_watcher() i=2 2026-03-20T11:46:50.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.619+0000 7fdbacf23900 20 remove_watcher() i=3 2026-03-20T11:46:50.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.619+0000 7fdbacf23900 20 remove_watcher() i=1 2026-03-20T11:46:50.616 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:50.619+0000 7fdbacf23900 20 remove_watcher() i=7 2026-03-20T11:46:55.623 INFO:rgw_multi.tests:trimming on zone=a2 2026-03-20T11:46:55.623 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '50', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:46:55.623 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '50', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:46:55.624 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bilog autotrim --rgw-sync-log-trim-max-buckets 50 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 20 --debug-ms 0 2026-03-20T11:46:55.661 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:46:55.661 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:46:55.683 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.686+0000 7fa44df93900 20 rados->read ofs=0 len=0 2026-03-20T11:46:55.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.687+0000 7fa44df93900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:46:55.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.687+0000 7fa44df93900 20 rados->read ofs=0 len=0 2026-03-20T11:46:55.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.688+0000 7fa44df93900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:46:55.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.688+0000 7fa44df93900 20 realm test-realm 6a789b89-c2b8-45f5-82cc-fb80234d0c6c 2026-03-20T11:46:55.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.688+0000 7fa44df93900 20 rados->read ofs=0 len=0 2026-03-20T11:46:55.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.688+0000 7fa44df93900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:46:55.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.688+0000 7fa44df93900 20 rados->read ofs=0 len=0 2026-03-20T11:46:55.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.689+0000 7fa44df93900 20 rados_obj.operate() r=0 bl.length=10 2026-03-20T11:46:55.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.689+0000 7fa44df93900 20 rados->read ofs=0 len=0 2026-03-20T11:46:55.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.689+0000 7fa44df93900 20 rados_obj.operate() r=0 bl.length=2020 2026-03-20T11:46:55.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.689+0000 7fa44df93900 20 rados->read ofs=0 len=0 2026-03-20T11:46:55.687 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.690+0000 7fa44df93900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:46:55.687 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.690+0000 7fa44df93900 20 rados->read ofs=0 len=0 2026-03-20T11:46:55.687 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.690+0000 7fa44df93900 20 rados_obj.operate() r=0 bl.length=1030 2026-03-20T11:46:55.687 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.690+0000 7fa44df93900 20 zone a2 found 2026-03-20T11:46:55.687 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.690+0000 7fa44df93900 4 Realm: test-realm (6a789b89-c2b8-45f5-82cc-fb80234d0c6c) 2026-03-20T11:46:55.687 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.690+0000 7fa44df93900 4 ZoneGroup: a (a) 2026-03-20T11:46:55.687 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.690+0000 7fa44df93900 4 Zone: a2 (51055576-6662-409d-8abb-ecaeefd3dd80) 2026-03-20T11:46:55.687 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.690+0000 7fa44df93900 4 using period configuration: cb2fcdce-2c1b-429a-8601-a284ee3a2381:12 2026-03-20T11:46:55.687 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.690+0000 7fa44df93900 20 period zonegroup name a 2026-03-20T11:46:55.687 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.690+0000 7fa44df93900 20 using current period zonegroup a 2026-03-20T11:46:55.687 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.690+0000 7fa44df93900 20 rados->read ofs=0 len=0 2026-03-20T11:46:55.688 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.691+0000 7fa44df93900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:55.688 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.691+0000 7fa44df93900 20 rados->read ofs=0 len=0 2026-03-20T11:46:55.688 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.691+0000 7fa44df93900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:55.688 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.691+0000 7fa44df93900 20 init(): pipe_map (zone=4b634ebf-58f5-45bc-839a-cb4b425e773a bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:46:55.688 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.691+0000 7fa44df93900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:55.688 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.691+0000 7fa44df93900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:55.688 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.691+0000 7fa44df93900 20 rados->read ofs=0 len=0 2026-03-20T11:46:55.688 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.691+0000 7fa44df93900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:55.688 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.691+0000 7fa44df93900 20 rados->read ofs=0 len=0 2026-03-20T11:46:55.688 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.692+0000 7fa44df93900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:55.688 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.692+0000 7fa44df93900 20 init(): pipe_map (zone=51055576-6662-409d-8abb-ecaeefd3dd80 bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:46:55.688 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.692+0000 7fa44df93900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:55.688 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.692+0000 7fa44df93900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:55.688 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.692+0000 7fa44df93900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:55.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.692+0000 7fa44df93900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:55.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.692+0000 7fa44df93900 20 started sync module instance, tier type = 2026-03-20T11:46:55.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.692+0000 7fa44df93900 20 generating connection object for zone a1 id 4b634ebf-58f5-45bc-839a-cb4b425e773a 2026-03-20T11:46:55.689 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.692+0000 7fa44df93900 20 started zone id=51055576-6662-409d-8abb-ecaeefd3dd80 (name=a2) with tier type = 2026-03-20T11:46:55.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.697+0000 7fa44df93900 20 add_watcher() i=1 2026-03-20T11:46:55.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.697+0000 7fa44df93900 20 add_watcher() i=0 2026-03-20T11:46:55.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.698+0000 7fa44df93900 20 add_watcher() i=7 2026-03-20T11:46:55.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.698+0000 7fa44df93900 20 add_watcher() i=3 2026-03-20T11:46:55.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.698+0000 7fa44df93900 20 add_watcher() i=6 2026-03-20T11:46:55.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.698+0000 7fa44df93900 20 add_watcher() i=4 2026-03-20T11:46:55.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.698+0000 7fa44df93900 20 add_watcher() i=5 2026-03-20T11:46:55.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.698+0000 7fa44df93900 20 add_watcher() i=2 2026-03-20T11:46:55.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.698+0000 7fa44df93900 2 all 8 watchers are set, enabling cache 2026-03-20T11:46:55.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.700+0000 7fa44df93900 20 rgw_check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2026-03-20T11:46:55.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.700+0000 7fa44df93900 20 rgw_check_secure_mon_conn(): mode 1 is insecure 2026-03-20T11:46:55.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.701+0000 7fa44df93900 5 note: GC not initialized 2026-03-20T11:46:55.866 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.869+0000 7fa44df93900 20 init_complete bucket index max shards: 11 2026-03-20T11:46:55.866 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.869+0000 7fa44df93900 20 Filter name: none 2026-03-20T11:46:55.866 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.869+0000 7fa3f27fc640 20 reqs_thread_entry: start 2026-03-20T11:46:55.866 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.869+0000 7fa3f0ff9640 20 reqs_thread_entry: start 2026-03-20T11:46:55.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.870+0000 7fa44df93900 10 rgw bucket trim manager: Watching bilog.trim 2026-03-20T11:46:55.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.870+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07852ae0:12BucketTrimCR: operate() 2026-03-20T11:46:55.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.870+0000 7fa44df93900 10 fetching active bucket counters 2026-03-20T11:46:55.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.870+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07855200:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:55.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.870+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07855200:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:55.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.870+0000 7fa44df93900 20 run: stack=0x558d078528a0 is io blocked 2026-03-20T11:46:55.867 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.870+0000 7fa4257fa640 20 trim: get_bucket_counters: [] 2026-03-20T11:46:55.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.871+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07855200:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:55.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.871+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07855200:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:55.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.871+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07855200:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:55.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.871+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07855200:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:55.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.871+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07852ae0:12BucketTrimCR: operate() 2026-03-20T11:46:55.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.871+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:55.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.871+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:55.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.871+0000 7fa44df93900 20 run: stack=0x558d078528a0 is io blocked 2026-03-20T11:46:55.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.871+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:55.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.871+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:55.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.871+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:55.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.871+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:55.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.871+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07852ae0:12BucketTrimCR: operate() 2026-03-20T11:46:55.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.871+0000 7fa44df93900 10 listing cold buckets from marker=16:69e609d8:root::dipepu-52:head 2026-03-20T11:46:55.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.871+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:14MetadataListCR: operate() 2026-03-20T11:46:55.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.871+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:14MetadataListCR: operate() 2026-03-20T11:46:55.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.871+0000 7fa44df93900 20 enqueued request req=0x7fa404002b20 2026-03-20T11:46:55.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.871+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:55.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.871+0000 7fa44df93900 20 req: 0x7fa404002b20 2026-03-20T11:46:55.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.871+0000 7fa44df93900 20 run: stack=0x558d078528a0 is io blocked 2026-03-20T11:46:55.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.871+0000 7fa4117fa640 20 dequeued request req=0x7fa404002b20 2026-03-20T11:46:55.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.871+0000 7fa4117fa640 20 RGWWQ: empty 2026-03-20T11:46:55.868 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.871+0000 7fa4117fa640 20 rgw async rados processor: starting metadata listing at 16:69e609d8:root::dipepu-52:head 2026-03-20T11:46:55.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.875+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-52 2026-03-20T11:46:55.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.875+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-32 2026-03-20T11:46:55.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.875+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-42 2026-03-20T11:46:55.872 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.875+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:46:55.876 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.879+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:46:55.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.882+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-7 2026-03-20T11:46:55.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.882+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-54 2026-03-20T11:46:55.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.882+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-49 2026-03-20T11:46:55.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.882+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-47 2026-03-20T11:46:55.879 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.882+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:46:55.882 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.886+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:46:55.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.889+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-11 2026-03-20T11:46:55.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.889+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-12 2026-03-20T11:46:55.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.889+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-41 2026-03-20T11:46:55.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.889+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-23 2026-03-20T11:46:55.885 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.889+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-15 2026-03-20T11:46:55.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.889+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-45 2026-03-20T11:46:55.886 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.889+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:46:55.889 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.892+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:46:55.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.895+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-9 2026-03-20T11:46:55.892 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.895+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:46:55.894 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.898+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-29 2026-03-20T11:46:55.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.898+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:46:55.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.900+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-31 2026-03-20T11:46:55.897 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.900+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:46:55.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.902+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-33 2026-03-20T11:46:55.899 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.902+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:55.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.905+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-4 2026-03-20T11:46:55.901 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.905+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:46:55.904 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.907+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:55.906 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.909+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:55.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.911+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-43 2026-03-20T11:46:55.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.911+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-30 2026-03-20T11:46:55.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.911+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-18 2026-03-20T11:46:55.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.911+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-2 2026-03-20T11:46:55.908 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.911+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:46:55.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.914+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-8 2026-03-20T11:46:55.911 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.914+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:46:55.913 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.916+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:55.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.918+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-58 2026-03-20T11:46:55.914 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.918+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:46:55.916 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.919+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:55.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.921+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-27 2026-03-20T11:46:55.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.921+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-28 2026-03-20T11:46:55.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.921+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-59 2026-03-20T11:46:55.918 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.921+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:46:55.919 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.922+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:55.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.924+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-40 2026-03-20T11:46:55.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.924+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-22 2026-03-20T11:46:55.921 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.924+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:46:55.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.925+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-56 2026-03-20T11:46:55.922 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.925+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:46:55.923 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.926+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:46:55.924 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.927+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:55.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.928+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-53 2026-03-20T11:46:55.925 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.928+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:46:55.926 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.929+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:46:55.927 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.930+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:46:55.928 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.931+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:46:55.929 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.932+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:46:55.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.933+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-34 2026-03-20T11:46:55.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.933+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-21 2026-03-20T11:46:55.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.933+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-14 2026-03-20T11:46:55.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.933+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-50 2026-03-20T11:46:55.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.933+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-57 2026-03-20T11:46:55.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.933+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:46:55.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.933+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-39 2026-03-20T11:46:55.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.933+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:55.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.933+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-37 2026-03-20T11:46:55.930 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.933+0000 7fa4117fa640 20 rgw async rados processor: restarting metadata listing 2026-03-20T11:46:55.938 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.941+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-10 2026-03-20T11:46:55.940 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.941+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:55.944 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.948+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:55.951 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.954+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:55.958 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.961+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:55.964 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.967+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:55.971 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.975+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:55.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.982+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-19 2026-03-20T11:46:55.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.982+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-3 2026-03-20T11:46:55.979 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.982+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:55.985 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.988+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:55.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.994+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-17 2026-03-20T11:46:55.991 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:55.994+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:55.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.000+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-38 2026-03-20T11:46:55.997 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.000+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:56.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.006+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-24 2026-03-20T11:46:56.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.006+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-36 2026-03-20T11:46:56.003 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.006+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:56.009 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.012+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:56.015 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.018+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:56.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.023+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-44 2026-03-20T11:46:56.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.023+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-35 2026-03-20T11:46:56.020 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.023+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:56.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.030+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-16 2026-03-20T11:46:56.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.030+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:56.032 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.035+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:56.037 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.041+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:56.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.046+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-55 2026-03-20T11:46:56.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.046+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-13 2026-03-20T11:46:56.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.046+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-48 2026-03-20T11:46:56.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.046+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-25 2026-03-20T11:46:56.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.046+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-51 2026-03-20T11:46:56.043 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.046+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:56.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa4117fa640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:56.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:14MetadataListCR: operate() 2026-03-20T11:46:56.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:14MetadataListCR: operate() 2026-03-20T11:46:56.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:14MetadataListCR: operate() 2026-03-20T11:46:56.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:14MetadataListCR: operate() 2026-03-20T11:46:56.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07852ae0:12BucketTrimCR: operate() 2026-03-20T11:46:56.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 4 collected 50 buckets for trim 2026-03-20T11:46:56.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078541f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 4 starting trim on bucket=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:46:56.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07857ab0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 4 starting trim on bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:46:56.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 cr:s=0x558d07776860:op=0x558d07858d80:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 4 starting trim on bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0785a050:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 4 starting trim on bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0785b320:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d0785ba70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 cr:s=0x558d07776860:op=0x558d0785c1c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0785c910:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0785b320:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 enqueued request req=0x558d0785d060 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 req: 0x558d0785d060 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 run: stack=0x558d07878200 is io blocked 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d0785ba70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 enqueued request req=0x558d0785d400 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 req: 0x558d0785d060 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 req: 0x558d0785d400 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 run: stack=0x558d0789c890 is io blocked 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 cr:s=0x558d07776860:op=0x558d0785c1c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 enqueued request req=0x558d0785d820 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 req: 0x558d0785d060 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 req: 0x558d0785d400 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 req: 0x558d0785d820 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 run: stack=0x558d07776860 is io blocked 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0785c910:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 enqueued request req=0x558d0785dc40 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 req: 0x558d0785d060 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 req: 0x558d0785d400 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 req: 0x558d0785d820 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 req: 0x558d0785dc40 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa44df93900 20 run: stack=0x558d07856e80 is io blocked 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa410ff9640 20 dequeued request req=0x558d0785d060 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa410ff9640 20 RGWWQ: 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa410ff9640 20 req: 0x558d0785d400 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa410ff9640 20 req: 0x558d0785d820 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa410ff9640 20 req: 0x558d0785dc40 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa410ff9640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 : miss 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa410ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa40bfff640 20 dequeued request req=0x558d0785d400 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa40bfff640 20 RGWWQ: 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa40bfff640 20 req: 0x558d0785d820 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa40bfff640 20 req: 0x558d0785dc40 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa40bfff640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 : miss 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa40bfff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa40b7fe640 20 dequeued request req=0x558d0785d820 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa40b7fe640 20 RGWWQ: 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa40b7fe640 20 req: 0x558d0785dc40 2026-03-20T11:46:56.049 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa40b7fe640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 : miss 2026-03-20T11:46:56.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa40b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa3f8fe9640 20 dequeued request req=0x558d0785dc40 2026-03-20T11:46:56.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa3f8fe9640 20 RGWWQ: empty 2026-03-20T11:46:56.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa3f8fe9640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 : miss 2026-03-20T11:46:56.050 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.051+0000 7fa3f8fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa40bfff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa40bfff640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 info.flags=0x17 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa40bfff640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 to cache LRU end 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa40bfff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa40bfff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa40bfff640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa40bfff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa40bfff640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-48 : miss 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa40bfff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa40b7fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa40b7fe640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 info.flags=0x17 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa40b7fe640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 to cache LRU end 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa40b7fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa40b7fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa40b7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa40b7fe640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-59 : miss 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa40b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa410ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=387 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa410ff9640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 info.flags=0x17 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa410ff9640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 to cache LRU end 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa410ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa410ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa410ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa410ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-61 : miss 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa410ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa3f8fe9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa3f8fe9640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 info.flags=0x17 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa3f8fe9640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 to cache LRU end 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa3f8fe9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa3f8fe9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa3f8fe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:46:56.051 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa3f8fe9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-21 : miss 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa3f8fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa40b7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa40b7fe640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-59 info.flags=0x0 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa40b7fe640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-59 to cache LRU end 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa40b7fe640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-59 : miss 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa40b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa3f8fe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa3f8fe9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-21 info.flags=0x0 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa3f8fe9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-21 to cache LRU end 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa3f8fe9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-21 : miss 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa3f8fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa410ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa410ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-61 info.flags=0x0 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa410ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-61 to cache LRU end 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa410ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-61 : miss 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa410ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa40bfff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa40bfff640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-48 info.flags=0x0 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa40bfff640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-48 to cache LRU end 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa40bfff640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-48 : miss 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.052+0000 7fa40bfff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa410ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa410ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-61 info.flags=0x0 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa410ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-61 to cache LRU end 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa410ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21): adding source pipe: {s={b=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa410ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21): adding dest pipe: {s={b=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa410ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa410ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa410ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa40bfff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa40bfff640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-48 info.flags=0x0 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa40bfff640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-48 to cache LRU end 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0785b320:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa40bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8): adding source pipe: {s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0785b320:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0785b320:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0785b320:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078541f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa40bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8): adding dest pipe: {s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d0785e060:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078541f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 run: stack=0x558d07878200 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d0785b320:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d0785b320:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa40bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa40bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa40bfff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = e0a82533b4dac94e88b1d5b5b5dfa4ade7be3faa415dc7721e5eb6c3a567538f 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:e0a82533b4dac94e88b1d5b5b5dfa4ade7be3faa415dc7721e5eb6c3a567538f 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 498f6a30f8bb1aef18c4e4cd0a40533465d5bfa0dffa39362d35cb5744e9db24 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=498f6a30f8bb1aef18c4e4cd0a40533465d5bfa0dffa39362d35cb5744e9db24 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=0, curl_handle=0x558d0785fe10 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 run: stack=0x558d07855790 is io blocked 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d0785ba70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d0785ba70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d0785ba70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d0785ba70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07857ab0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d0785c060:op=0x558d07862370:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07857ab0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.053 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 run: stack=0x558d0789c890 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d0785c060:op=0x558d078629c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d0785c060:op=0x558d078629c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = b9ea00a4a6d7b9f990092071e9e3ef600723cfd389366a0cf095cdcdc1e34133 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:b9ea00a4a6d7b9f990092071e9e3ef600723cfd389366a0cf095cdcdc1e34133 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = b623901f1174a5de6404968851be50c4d5407d28b347229bc63742a7d4d53c67 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b623901f1174a5de6404968851be50c4d5407d28b347229bc63742a7d4d53c67 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=1, curl_handle=0x558d07864550 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 run: stack=0x558d0785c060 is io blocked 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa40b7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa40b7fe640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-59 info.flags=0x0 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa40b7fe640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-59 to cache LRU end 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa40b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19): adding source pipe: {s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa40b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19): adding dest pipe: {s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa40b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa40b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa40b7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d07776860:op=0x558d0785c1c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d07776860:op=0x558d0785c1c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d07776860:op=0x558d0785c1c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d07776860:op=0x558d0785c1c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d07776860:op=0x558d07858d80:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d07866440:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d07776860:op=0x558d07858d80:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 run: stack=0x558d07776860 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d07866a60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d07866a60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.054 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 78c5880d0faba87cf358abb6883181c1d05665958edb610c651f0245a9b32c5a 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:78c5880d0faba87cf358abb6883181c1d05665958edb610c651f0245a9b32c5a 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 9ec3ec4d4530e3770b7e0155071fa726e9a50168eebcfa175a4836c90d4adb13 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=9ec3ec4d4530e3770b7e0155071fa726e9a50168eebcfa175a4836c90d4adb13 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=2, curl_handle=0x558d07868640 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 run: stack=0x558d0785c780 is io blocked 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa3f8fe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa3f8fe9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-21 info.flags=0x0 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa3f8fe9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-21 to cache LRU end 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa3f8fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26): adding source pipe: {s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa3f8fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26): adding dest pipe: {s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa3f8fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa3f8fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa3f8fe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0785c910:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0785c910:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0785c910:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0785c910:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0785a050:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d0786a4d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.055 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0785a050:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 run: stack=0x558d07856e80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d0786ac80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d0786ac80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 0421f3c60dd8e3bf02c5371b372b78f2962a1d890d72e996dbd537be07832c31 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:0421f3c60dd8e3bf02c5371b372b78f2962a1d890d72e996dbd537be07832c31 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 955e5b140fe77d57a94e43db60f1a7027241db1baed00866577151a00fdb2633 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=955e5b140fe77d57a94e43db60f1a7027241db1baed00866577151a00fdb2633 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=3, curl_handle=0x558d0786c860 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa44df93900 20 run: stack=0x558d0786aaf0 is io blocked 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa3f0ff9640 20 link_request req_data=0x558d07860910 req_data->id=0, curl_handle=0x558d0785fe10 2026-03-20T11:46:56.056 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa3f0ff9640 20 link_request req_data=0x558d07864ca0 req_data->id=1, curl_handle=0x558d07864550 2026-03-20T11:46:56.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa3f0ff9640 20 link_request req_data=0x558d07868d50 req_data->id=2, curl_handle=0x558d07868640 2026-03-20T11:46:56.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.053+0000 7fa3f0ff9640 20 link_request req_data=0x558d0786cf70 req_data->id=3, curl_handle=0x558d0786c860 2026-03-20T11:46:56.057 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:56.054+0000 7f125799f640 -1 req 1931721944081383910 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:56.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa44df93900 20 cr:s=0x558d0785c060:op=0x558d078629c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa44df93900 20 cr:s=0x558d0785c060:op=0x558d078629c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:56.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa44df93900 20 cr:s=0x558d0785c060:op=0x558d07862370:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa44df93900 15 stack 0x558d0785c060 end 2026-03-20T11:46:56.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa44df93900 20 run: stack=0x558d0785c060 is done 2026-03-20T11:46:56.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07857ab0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa44df93900 20 collect(): s=0x558d0789c890 stack=0x558d0785c060 is complete 2026-03-20T11:46:56.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa44df93900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:56.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa44df93900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:56.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07857ab0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:56.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa44df93900 15 stack 0x558d0789c890 end 2026-03-20T11:46:56.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa44df93900 20 stack->operate() returned ret=-22 2026-03-20T11:46:56.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa44df93900 20 run: stack=0x558d0789c890 is done 2026-03-20T11:46:56.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa44df93900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:56.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07857ab0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.058 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa44df93900 4 starting trim on bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07862350:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07862350:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa44df93900 20 enqueued request req=0x558d0785d400 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa44df93900 20 req: 0x558d0785d400 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa44df93900 20 run: stack=0x558d0789c890 is io blocked 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa40a7fc640 20 dequeued request req=0x558d0785d400 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa40a7fc640 20 RGWWQ: empty 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa40a7fc640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 : miss 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.056+0000 7fa40a7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.057+0000 7fa40a7fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.057+0000 7fa40a7fc640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 info.flags=0x17 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.057+0000 7fa40a7fc640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 to cache LRU end 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.057+0000 7fa40a7fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.057+0000 7fa40a7fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.057+0000 7fa40a7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.057+0000 7fa40a7fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-41 : miss 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.057+0000 7fa40a7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.057+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d07866a60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.057+0000 7fa40a7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.057+0000 7fa40a7fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-41 info.flags=0x0 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.057+0000 7fa40a7fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-41 to cache LRU end 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.057+0000 7fa40a7fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-41 : miss 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.057+0000 7fa40a7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.057+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d07866a60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.057+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d0785b320:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.057+0000 7fa40a7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.057+0000 7fa40a7fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-41 info.flags=0x0 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.057+0000 7fa40a7fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-41 to cache LRU end 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa40a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1): adding source pipe: {s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa40a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1): adding dest pipe: {s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d0786ac80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa40a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa40a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa40a7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d07866a60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d0785b320:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d0786ac80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07862350:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d07866a60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d0785b320:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.059 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d0786ac80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07862350:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d07866440:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 15 stack 0x558d0785c780 end 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 run: stack=0x558d0785c780 is done 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d0785b320:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d0786ac80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07862350:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d07776860:op=0x558d07858d80:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 collect(): s=0x558d07776860 stack=0x558d0785c780 is complete 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 10 trimming bilogs for bucket=:dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) markers=[,00000000001.65.6,,,,00000000001.44.6,,,00000000001.58.6,,00000000001.63.6], shards=11 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d0785e060:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 15 stack 0x558d07855790 end 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 run: stack=0x558d07855790 is done 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d0786a4d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 15 stack 0x558d0786aaf0 end 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 run: stack=0x558d0786aaf0 is done 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07862350:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d07776860:op=0x558d078a4c30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 10 trimming bilog shard 1 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.65.6 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 10 trimming bilog shard 5 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.44.6 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 10 trimming bilog shard 8 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.58.6 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 10 trimming bilog shard 10 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.63.6 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 run: stack=0x558d07776860 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078541f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 collect(): s=0x558d07878200 stack=0x558d07855790 is complete 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0785a050:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 collect(): s=0x558d07856e80 stack=0x558d0786aaf0 is complete 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 10 trimming bilogs for bucket=:dipepu-21[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26]) markers=[,,,,,,,,00000000005.84.14,,], shards=11 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07857ab0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d078a9fd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d07860530:op=0x558d078ad1b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d078648e0:op=0x558d078a7690:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d078ab180:op=0x558d078a07e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0786a4d0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.0 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.1 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.2 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.3 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.4 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.5 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.6 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.7 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.8 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.9 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.10 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 run: stack=0x558d07878200 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0785b320:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 10 trimming bilog shard 8 of :dipepu-21[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26]) at marker 00000000005.84.14 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 run: stack=0x558d07856e80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078ab820:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07857ab0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 run: stack=0x558d0789c890 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d078a9fd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.1 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 run: stack=0x558d0785c780 is io blocked 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d07860530:op=0x558d078ad1b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.5 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 run: stack=0x558d07860530 is io blocked 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d078648e0:op=0x558d078a7690:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.8 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 run: stack=0x558d078648e0 is io blocked 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d078ab180:op=0x558d078a07e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.10 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 run: stack=0x558d078ab180 is io blocked 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d07872a80:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d078ae5b0:op=0x558d078a83b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d078a1ec0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d07875380:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d078a6820:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a6e10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0786ee10:op=0x558d078a3060:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0785f4f0:op=0x558d078a3650:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d0786ac80:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0786e6f0:op=0x558d0786b270:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d07874cb0:op=0x558d0785dfc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d0785e5b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d07862350:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d07872a80:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 run: stack=0x558d07855790 is io blocked 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d078ae5b0:op=0x558d078a83b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 run: stack=0x558d078ae5b0 is io blocked 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d078a1ec0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 run: stack=0x558d078a1890 is io blocked 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d07875380:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 run: stack=0x558d07372f40 is io blocked 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d078a6820:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 run: stack=0x558d07860730 is io blocked 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a6e10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is io blocked 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0786ee10:op=0x558d078a3060:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 run: stack=0x558d0786ee10 is io blocked 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0785f4f0:op=0x558d078a3650:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 run: stack=0x558d0785f4f0 is io blocked 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d0786ac80:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is io blocked 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0786e6f0:op=0x558d0786b270:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 run: stack=0x558d0786e6f0 is io blocked 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d07874cb0:op=0x558d0785dfc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 run: stack=0x558d07874cb0 is io blocked 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d0785e5b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26.8 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 run: stack=0x558d07876860 is io blocked 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d07862350:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.061 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 4582c2941ae57e666f94475cedb8312dc881d3bacae8bf5c63428143207d4c05 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:4582c2941ae57e666f94475cedb8312dc881d3bacae8bf5c63428143207d4c05 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 4426827d3b997ba5c2e15baf754628e2183b5ca56d88bf8da7824ed9ea191ba1 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=4426827d3b997ba5c2e15baf754628e2183b5ca56d88bf8da7824ed9ea191ba1 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=4, curl_handle=0x558d0786c860 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.058+0000 7fa44df93900 20 run: stack=0x558d0786aaf0 is io blocked 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.059+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d078a6820:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.059+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d078a6820:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.059+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d078a6820:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.059+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d078a6820:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.059+0000 7fa44df93900 15 stack 0x558d07860730 end 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.059+0000 7fa44df93900 20 run: stack=0x558d07860730 is done 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.059+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0786a4d0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.059+0000 7fa44df93900 20 run: stack=0x558d07878200 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d07872a80:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d07872a80:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d0785e5b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d0785e5b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 15 stack 0x558d07876860 end 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 run: stack=0x558d07876860 is done 2026-03-20T11:46:56.062 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d07872a80:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0785b320:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d07872a80:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 15 stack 0x558d07855790 end 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 run: stack=0x558d07855790 is done 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0785a050:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 trim: trimmed bucket instance dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 15 stack 0x558d07856e80 end 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 run: stack=0x558d07856e80 is done 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0786a4d0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 run: stack=0x558d07878200 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0785a050:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 4 starting trim on bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0785e5b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0785e5b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 enqueued request req=0x558d0785d820 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 req: 0x558d0785d820 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 run: stack=0x558d07856e80 is io blocked 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa409ffb640 20 dequeued request req=0x558d0785d820 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa409ffb640 20 RGWWQ: empty 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa409ffb640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 : miss 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa409ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d0786ac80:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d0786ac80:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d0786ac80:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d0786ac80:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 15 stack 0x558d0786bfc0 end 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is done 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0786a4d0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.060+0000 7fa44df93900 20 run: stack=0x558d07878200 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.063 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa3f0ff9640 20 link_request req_data=0x558d0786cf70 req_data->id=4, curl_handle=0x558d0786c860 2026-03-20T11:46:56.063 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:56.061+0000 7f128ea0d640 -1 req 7261133533656909022 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa409ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa409ffb640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 info.flags=0x17 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa409ffb640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 to cache LRU end 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa409ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa409ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa409ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa409ffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-49 : miss 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa409ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 20 cr:s=0x558d078ab180:op=0x558d078a07e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 20 cr:s=0x558d078ab180:op=0x558d078a07e0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 15 stack 0x558d078ab180 end 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 20 run: stack=0x558d078ab180 is done 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 20 cr:s=0x558d07776860:op=0x558d078a4c30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 20 run: stack=0x558d07776860 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d07875380:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 20 cr:s=0x558d078648e0:op=0x558d078a7690:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 20 cr:s=0x558d078648e0:op=0x558d078a7690:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 15 stack 0x558d078648e0 end 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 20 run: stack=0x558d078648e0 is done 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d07875380:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 20 cr:s=0x558d07776860:op=0x558d078a4c30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 20 run: stack=0x558d07776860 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d07875380:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d07875380:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 15 stack 0x558d07372f40 end 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 20 run: stack=0x558d07372f40 is done 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0786a4d0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:56.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 20 run: stack=0x558d07878200 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d078a1ec0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d078a1ec0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d078a1ec0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d078a1ec0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 15 stack 0x558d078a1890 end 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 20 run: stack=0x558d078a1890 is done 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0786a4d0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.061+0000 7fa44df93900 20 run: stack=0x558d07878200 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.062+0000 7fa409ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.062+0000 7fa409ffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-49 info.flags=0x0 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.062+0000 7fa409ffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-49 to cache LRU end 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.062+0000 7fa409ffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-49 : miss 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.062+0000 7fa409ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.062+0000 7fa44df93900 20 cr:s=0x558d078ae5b0:op=0x558d078a83b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.062+0000 7fa44df93900 20 cr:s=0x558d078ae5b0:op=0x558d078a83b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.062+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d078a9fd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.062+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d078a9fd0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.062+0000 7fa44df93900 15 stack 0x558d0785c780 end 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.062+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.062+0000 7fa44df93900 20 run: stack=0x558d0785c780 is done 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.062+0000 7fa44df93900 20 cr:s=0x558d078ae5b0:op=0x558d078a83b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.062+0000 7fa44df93900 20 cr:s=0x558d07776860:op=0x558d078a4c30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.062+0000 7fa44df93900 20 run: stack=0x558d07776860 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.062+0000 7fa44df93900 20 cr:s=0x558d078ae5b0:op=0x558d078a83b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.062+0000 7fa44df93900 15 stack 0x558d078ae5b0 end 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 run: stack=0x558d078ae5b0 is done 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0786a4d0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 run: stack=0x558d07878200 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa409ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa409ffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-49 info.flags=0x0 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa409ffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-49 to cache LRU end 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa409ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9): adding source pipe: {s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa409ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9): adding dest pipe: {s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d0786ee10:op=0x558d078a3060:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d0786ee10:op=0x558d078a3060:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d0786ee10:op=0x558d078a3060:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d0786ee10:op=0x558d078a3060:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 15 stack 0x558d0786ee10 end 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 run: stack=0x558d0786ee10 is done 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0786a4d0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 run: stack=0x558d07878200 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa409ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa409ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa409ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0785e5b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0785e5b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d07860530:op=0x558d078ad1b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d07860530:op=0x558d078ad1b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 15 stack 0x558d07860530 end 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 run: stack=0x558d07860530 is done 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0785e5b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a6e10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d07776860:op=0x558d078a4c30:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.065 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0785e5b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a6e10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d07776860:op=0x558d07858d80:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 trim: trimmed bucket instance dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 15 stack 0x558d07776860 end 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 run: stack=0x558d07776860 is done 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0785a050:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a6e10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d0786ac50:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0785a050:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 run: stack=0x558d07856e80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a6e10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 15 stack 0x558d0785c1a0 end 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is done 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d07776860:op=0x558d07858d80:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 4 starting trim on bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d078a1e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d0785f4f0:op=0x558d078a3650:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0786a4d0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 run: stack=0x558d07878200 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d07776860:op=0x558d0785e5b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d078a1e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = e2d8daaa60064b9e64dd521caca9fa2639bc31a9429a9cceafc42ac98715ce28 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:e2d8daaa60064b9e64dd521caca9fa2639bc31a9429a9cceafc42ac98715ce28 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 5716e34d571cf32de9e62e234a8abeea298830347887c1a8ffb56df5035754bf 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5716e34d571cf32de9e62e234a8abeea298830347887c1a8ffb56df5035754bf 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=5, curl_handle=0x558d0785fe10 2026-03-20T11:46:56.066 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 run: stack=0x558d0785c780 is io blocked 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d0785f4f0:op=0x558d078a3650:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d07776860:op=0x558d0785e5b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 enqueued request req=0x558d07863d20 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 req: 0x558d07863d20 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 run: stack=0x558d07776860 is io blocked 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d07874cb0:op=0x558d0785dfc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d0785f4f0:op=0x558d078a3650:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d07874cb0:op=0x558d0785dfc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d0785f4f0:op=0x558d078a3650:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 15 stack 0x558d0785f4f0 end 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 run: stack=0x558d0785f4f0 is done 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d07874cb0:op=0x558d0785dfc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0786a4d0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 run: stack=0x558d07878200 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d07874cb0:op=0x558d0785dfc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 15 stack 0x558d07874cb0 end 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa4097fa640 20 dequeued request req=0x558d07863d20 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa4097fa640 20 RGWWQ: empty 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa4097fa640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 : miss 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa4097fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 run: stack=0x558d07874cb0 is done 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0786a4d0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 run: stack=0x558d07878200 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d0786e6f0:op=0x558d0786b270:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d0786e6f0:op=0x558d0786b270:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d0786e6f0:op=0x558d0786b270:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d0786e6f0:op=0x558d0786b270:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 15 stack 0x558d0786e6f0 end 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 run: stack=0x558d0786e6f0 is done 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0786a4d0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078541f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0785dfc0:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0785dfc0:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 enqueued request req=0x558d0785b950 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 req: 0x558d0785b950 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa44df93900 20 run: stack=0x558d07878200 is io blocked 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa3f0ff9640 20 link_request req_data=0x558d07860910 req_data->id=5, curl_handle=0x558d0785fe10 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa408ff9640 20 dequeued request req=0x558d0785b950 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa408ff9640 20 RGWWQ: empty 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa4097fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa4097fa640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 info.flags=0x17 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa4097fa640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 to cache LRU end 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa4097fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa4097fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa4097fa640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa4097fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa4097fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-45 : miss 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.063+0000 7fa4097fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa4097fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa4097fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-45 info.flags=0x0 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa4097fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-45 to cache LRU end 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa4097fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-45 : miss 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa4097fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa4097fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa4097fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-45 info.flags=0x0 2026-03-20T11:46:56.067 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa4097fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-45 to cache LRU end 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa4097fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5): adding source pipe: {s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa4097fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5): adding dest pipe: {s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa4097fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa4097fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa4097fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 20 cr:s=0x558d07776860:op=0x558d0785e5b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 20 cr:s=0x558d07776860:op=0x558d0785e5b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 20 cr:s=0x558d07776860:op=0x558d0785e5b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 20 cr:s=0x558d07776860:op=0x558d0785e5b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 20 cr:s=0x558d07776860:op=0x558d07858d80:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d07872a40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 20 cr:s=0x558d07776860:op=0x558d07858d80:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 20 run: stack=0x558d07776860 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = cdc6d21b55410f2fab073a477655ed4efdb7203a37dd7dae4b319e45e2793d8b 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:cdc6d21b55410f2fab073a477655ed4efdb7203a37dd7dae4b319e45e2793d8b 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = bd848c7f944885fc5a7aee096948a3de7aca61faec561a3a5b490e4b704badf0 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=bd848c7f944885fc5a7aee096948a3de7aca61faec561a3a5b490e4b704badf0 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=6, curl_handle=0x558d07868640 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is io blocked 2026-03-20T11:46:56.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.064+0000 7fa3f0ff9640 20 link_request req_data=0x558d07868d50 req_data->id=6, curl_handle=0x558d07868640 2026-03-20T11:46:56.068 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:56.065+0000 7f1222935640 -1 req 2344717593922682133 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.065+0000 7fa408ff9640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 info.flags=0x11 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.065+0000 7fa408ff9640 10 rgw async rados processor: moving a2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 to cache LRU end 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.065+0000 7fa408ff9640 10 rgw async rados processor: distributing notification oid=a2.rgw.control:notify.6 cni=[op: 0, obj: a2.rgw.meta:root:.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21, ofs0, ns] 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.066+0000 7fa4257fa640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 343597383951 cookie 94064205215232 notifier 6540 bl.length()=567 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.066+0000 7fa4257fa640 10 rgw watcher librados: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 info.flags=0x11 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.066+0000 7fa4257fa640 10 rgw watcher librados: moving a2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 to cache LRU end 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.066+0000 7fa408ff9640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-61[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21]): orig_sources= new_sources= 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.066+0000 7fa408ff9640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-61[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21]): potential sources added=[] removed=[] 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.066+0000 7fa408ff9640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-61[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21]): orig_dests= new_dests= 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.066+0000 7fa408ff9640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-61[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21]): potential dests added=[] removed=[] 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.066+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0785dfc0:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.066+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0785dfc0:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.066+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0785dfc0:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.066+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0785dfc0:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.066+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078541f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.066+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078a9fd0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.066+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078a9fd0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.066+0000 7fa44df93900 20 enqueued request req=0x558d07864ac0 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.066+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.066+0000 7fa44df93900 20 req: 0x558d07864ac0 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.066+0000 7fa44df93900 20 run: stack=0x558d07878200 is io blocked 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.066+0000 7fa403fff640 20 dequeued request req=0x558d07864ac0 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.066+0000 7fa403fff640 20 RGWWQ: empty 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.067+0000 7fa403fff640 10 rgw async rados processor: removing a2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 from cache 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.067+0000 7fa403fff640 10 rgw async rados processor: distributing notification oid=a2.rgw.control:notify.6 cni=[op: 1, obj: a2.rgw.meta:root:.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21, ofs0, ns] 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa4257fa640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 343597383952 cookie 94064205215232 notifier 6540 bl.length()=206 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078a9fd0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078a9fd0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078a9fd0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078a9fd0:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078541f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa44df93900 20 trim: trimmed bucket instance dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa44df93900 15 stack 0x558d07878200 end 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa44df93900 20 run: stack=0x558d07878200 is done 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078541f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa44df93900 4 starting trim on bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078a9fd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078a9fd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.069 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa44df93900 20 enqueued request req=0x558d07875c10 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa44df93900 20 req: 0x558d07875c10 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa44df93900 20 run: stack=0x558d07878200 is io blocked 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa4037fe640 20 dequeued request req=0x558d07875c10 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa4037fe640 20 RGWWQ: empty 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa4037fe640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 : miss 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa4037fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa4037fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa4037fe640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 info.flags=0x17 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa4037fe640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 to cache LRU end 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa4037fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa4037fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa4037fe640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa4037fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.068+0000 7fa4037fe640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-33 : miss 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa4037fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa4037fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa4037fe640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-33 info.flags=0x0 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa4037fe640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-33 to cache LRU end 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa4037fe640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-33 : miss 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa4037fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa4037fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa4037fe640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-33 info.flags=0x0 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa4037fe640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-33 to cache LRU end 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa4037fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38): adding source pipe: {s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa4037fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38): adding dest pipe: {s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa4037fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa4037fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa4037fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078a9fd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078a9fd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078a9fd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078a9fd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078541f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d0786a4d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078541f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 20 run: stack=0x558d07878200 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d0786c880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d0786c880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.070 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 393190a6d2117578d1acb0caeb7ca948f8181216ae7a10cf640b8d808021aa58 2026-03-20T11:46:56.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.071 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.071 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.071 INFO:teuthology.orchestra.run.vm02.stderr:393190a6d2117578d1acb0caeb7ca948f8181216ae7a10cf640b8d808021aa58 2026-03-20T11:46:56.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 6630f40dbd9a92cfce03d7405ff97b607b5a59bd41b165cb25f1e3d8682902d7 2026-03-20T11:46:56.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=6630f40dbd9a92cfce03d7405ff97b607b5a59bd41b165cb25f1e3d8682902d7 2026-03-20T11:46:56.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=7, curl_handle=0x558d07864550 2026-03-20T11:46:56.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa44df93900 20 run: stack=0x558d078a1890 is io blocked 2026-03-20T11:46:56.071 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.069+0000 7fa3f0ff9640 20 link_request req_data=0x7fa3ec005db0 req_data->id=7, curl_handle=0x558d07864550 2026-03-20T11:46:56.071 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:56.070+0000 7f12068fd640 -1 req 17418986741941240414 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:56.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d07862350:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d07862350:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078ab820:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa44df93900 15 stack 0x558d0786aaf0 end 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa44df93900 20 run: stack=0x558d0786aaf0 is done 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07857ab0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa44df93900 20 collect(): s=0x558d0789c890 stack=0x558d0786aaf0 is complete 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa44df93900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa44df93900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07857ab0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa44df93900 15 stack 0x558d0789c890 end 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa44df93900 20 stack->operate() returned ret=-22 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa44df93900 20 run: stack=0x558d0789c890 is done 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa44df93900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07857ab0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa44df93900 4 starting trim on bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078a9fd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078a9fd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa44df93900 20 enqueued request req=0x558d078ad9b0 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa44df93900 20 req: 0x558d078ad9b0 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa44df93900 20 run: stack=0x558d0789c890 is io blocked 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa402ffd640 20 dequeued request req=0x558d078ad9b0 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa402ffd640 20 RGWWQ: empty 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa402ffd640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 : miss 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa402ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa402ffd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa402ffd640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 info.flags=0x17 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa402ffd640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 to cache LRU end 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa402ffd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa402ffd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa402ffd640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa402ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa402ffd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-47 : miss 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.102+0000 7fa402ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa402ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa402ffd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-47 info.flags=0x0 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa402ffd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-47 to cache LRU end 2026-03-20T11:46:56.099 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa402ffd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-47 : miss 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa402ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa402ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa402ffd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-47 info.flags=0x0 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa402ffd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-47 to cache LRU end 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa402ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7): adding source pipe: {s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa402ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7): adding dest pipe: {s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa402ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa402ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa402ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078a9fd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078a9fd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078a9fd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078a9fd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07857ab0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078ab820:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07857ab0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 20 run: stack=0x558d0789c890 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078a9fd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078a9fd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = eba3695e4a31f221f1a3c554274307913471a964515c3f46380cf8346d0d0b7e 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.100 INFO:teuthology.orchestra.run.vm02.stderr:eba3695e4a31f221f1a3c554274307913471a964515c3f46380cf8346d0d0b7e 2026-03-20T11:46:56.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 6589301f72a926e18213dbaa0519c0a4415f5ea11d688cfbf3a7904d8ec62e98 2026-03-20T11:46:56.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=6589301f72a926e18213dbaa0519c0a4415f5ea11d688cfbf3a7904d8ec62e98 2026-03-20T11:46:56.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=8, curl_handle=0x558d0786c860 2026-03-20T11:46:56.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa44df93900 20 run: stack=0x558d0786aaf0 is io blocked 2026-03-20T11:46:56.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.103+0000 7fa3f0ff9640 20 link_request req_data=0x558d07864ca0 req_data->id=8, curl_handle=0x558d0786c860 2026-03-20T11:46:56.101 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:56.103+0000 7f12eeacd640 -1 req 10826304512532835735 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:56.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d078a1e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d078a1e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d078a1e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d078a1e70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d0786ac50:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa44df93900 15 stack 0x558d0785c780 end 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa44df93900 20 run: stack=0x558d0785c780 is done 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0785a050:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa44df93900 20 collect(): s=0x558d07856e80 stack=0x558d0785c780 is complete 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa44df93900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa44df93900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0785a050:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa44df93900 15 stack 0x558d07856e80 end 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa44df93900 20 stack->operate() returned ret=-22 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa44df93900 20 run: stack=0x558d07856e80 is done 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa44df93900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa44df93900 4 starting trim on bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078ad160:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078ad160:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa44df93900 20 enqueued request req=0x558d0785d060 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa44df93900 20 req: 0x558d0785d060 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa44df93900 20 run: stack=0x558d07856e80 is io blocked 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa4027fc640 20 dequeued request req=0x558d0785d060 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa4027fc640 20 RGWWQ: empty 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa4027fc640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 : miss 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa4027fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa4027fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa4027fc640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 info.flags=0x17 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa4027fc640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 to cache LRU end 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa4027fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa4027fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa4027fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa4027fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-58 : miss 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa4027fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa4027fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa4027fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-58 info.flags=0x0 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa4027fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-58 to cache LRU end 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa4027fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-58 : miss 2026-03-20T11:46:56.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.105+0000 7fa4027fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa4027fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa4027fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-58 info.flags=0x0 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa4027fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-58 to cache LRU end 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa4027fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18): adding source pipe: {s={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa4027fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18): adding dest pipe: {s={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa4027fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa4027fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa4027fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d07872a40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 15 stack 0x558d0785c1a0 end 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is done 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078ad160:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 cr:s=0x558d07776860:op=0x558d07858d80:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 collect(): s=0x558d07776860 stack=0x558d0785c1a0 is complete 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 cr:s=0x558d07776860:op=0x558d07858d80:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 15 stack 0x558d07776860 end 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 stack->operate() returned ret=-22 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 run: stack=0x558d07776860 is done 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078ad160:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078ad160:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 4 starting trim on bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078ad160:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.103 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d073d49b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d073d49b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 enqueued request req=0x558d0785d060 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 req: 0x558d0785d060 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is io blocked 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d078a1e20:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 run: stack=0x558d07856e80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa401ffb640 20 dequeued request req=0x558d0785d060 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa401ffb640 20 RGWWQ: empty 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa401ffb640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 : miss 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa401ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 9d9474251a21c2994dcfbfd3f8383791569fc8f9d13f2d0b099031de303311f1 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:9d9474251a21c2994dcfbfd3f8383791569fc8f9d13f2d0b099031de303311f1 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = c8c43b442d1045b108cc3f35fb228b7b4263fa355d76f47222be0010082aa1a7 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c8c43b442d1045b108cc3f35fb228b7b4263fa355d76f47222be0010082aa1a7 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=9, curl_handle=0x558d07868640 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa44df93900 20 run: stack=0x558d0785c780 is io blocked 2026-03-20T11:46:56.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa3f0ff9640 20 link_request req_data=0x558d0785d7e0 req_data->id=9, curl_handle=0x558d07868640 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa401ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa401ffb640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 info.flags=0x17 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa401ffb640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 to cache LRU end 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa401ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa401ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa401ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa401ffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-30 : miss 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.106+0000 7fa401ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa401ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa401ffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-30 info.flags=0x0 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa401ffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-30 to cache LRU end 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa401ffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-30 : miss 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa401ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa401ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa401ffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-30 info.flags=0x0 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa401ffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-30 to cache LRU end 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa401ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35): adding source pipe: {s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa401ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35): adding dest pipe: {s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa401ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa401ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa401ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d073d49b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d073d49b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d073d49b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d073d49b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078a4c10:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d073d49b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d073d49b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.105 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = e1874d9c9715276f02e368019336867945c6a51fb0a38e23ae8b2e0c52427512 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr:e1874d9c9715276f02e368019336867945c6a51fb0a38e23ae8b2e0c52427512 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 007ba877f3ab104739ff02e0bc56a1aeca49fedd26cfb8b9569cf5a561e08a29 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=007ba877f3ab104739ff02e0bc56a1aeca49fedd26cfb8b9569cf5a561e08a29 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=10, curl_handle=0x558d0785fe10 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa44df93900 20 run: stack=0x558d07372f40 is io blocked 2026-03-20T11:46:56.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.107+0000 7fa3f0ff9640 20 link_request req_data=0x558d07877c80 req_data->id=10, curl_handle=0x558d0785fe10 2026-03-20T11:46:56.106 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:56.108+0000 7f1210110640 -1 req 11383307889862866555 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:56.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d0786c880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d0786c880:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:56.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d0786a4d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa44df93900 15 stack 0x558d078a1890 end 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa44df93900 20 run: stack=0x558d078a1890 is done 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078541f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa44df93900 20 collect(): s=0x558d07878200 stack=0x558d078a1890 is complete 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa44df93900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa44df93900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078541f0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa44df93900 15 stack 0x558d07878200 end 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa44df93900 20 stack->operate() returned ret=-22 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa44df93900 20 run: stack=0x558d07878200 is done 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa44df93900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078541f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa44df93900 4 starting trim on bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa44df93900 20 enqueued request req=0x558d07875c10 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa44df93900 20 req: 0x558d07875c10 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa44df93900 20 run: stack=0x558d07878200 is io blocked 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa4017fa640 20 dequeued request req=0x558d07875c10 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa4017fa640 20 RGWWQ: empty 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa4017fa640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 : miss 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa4017fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa4017fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa4017fa640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 info.flags=0x17 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa4017fa640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 to cache LRU end 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa4017fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa4017fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa4017fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa4017fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-20 : miss 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa4017fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa4017fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa4017fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-20 info.flags=0x0 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa4017fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-20 to cache LRU end 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa4017fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-20 : miss 2026-03-20T11:46:56.108 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.111+0000 7fa4017fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa4017fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa4017fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-20 info.flags=0x0 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa4017fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-20 to cache LRU end 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa4017fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25): adding source pipe: {s={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa4017fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25): adding dest pipe: {s={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa4017fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa4017fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa4017fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078541f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07872a40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078541f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 20 run: stack=0x558d07878200 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d0786c880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d0786c880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.109 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.110 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 52ecb39dae0618e56ab9cc97e592b2c03c2355d448f8b84a234182ec4aa8c4a0 2026-03-20T11:46:56.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.110 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.110 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.110 INFO:teuthology.orchestra.run.vm02.stderr:52ecb39dae0618e56ab9cc97e592b2c03c2355d448f8b84a234182ec4aa8c4a0 2026-03-20T11:46:56.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 9ea0eca9f94071d84b692cef8d13e30c7f41cca210a4c6a1f9faf23878ac1dc0 2026-03-20T11:46:56.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=9ea0eca9f94071d84b692cef8d13e30c7f41cca210a4c6a1f9faf23878ac1dc0 2026-03-20T11:46:56.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=11, curl_handle=0x558d07864550 2026-03-20T11:46:56.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa44df93900 20 run: stack=0x558d078a1890 is io blocked 2026-03-20T11:46:56.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.112+0000 7fa3f0ff9640 20 link_request req_data=0x558d078522e0 req_data->id=11, curl_handle=0x558d07864550 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078a9fd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078a9fd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078ab820:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa44df93900 15 stack 0x558d0786aaf0 end 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa44df93900 20 run: stack=0x558d0786aaf0 is done 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07857ab0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa44df93900 20 collect(): s=0x558d0789c890 stack=0x558d0786aaf0 is complete 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa44df93900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa44df93900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07857ab0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa44df93900 15 stack 0x558d0789c890 end 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa44df93900 20 stack->operate() returned ret=-22 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa44df93900 20 run: stack=0x558d0789c890 is done 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa44df93900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07873360:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa44df93900 4 starting trim on bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d073dae30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d073dae30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa44df93900 20 enqueued request req=0x558d078ad9b0 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa44df93900 20 req: 0x558d078ad9b0 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa44df93900 20 run: stack=0x558d0789c890 is io blocked 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa400ff9640 20 dequeued request req=0x558d078ad9b0 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa400ff9640 20 RGWWQ: empty 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa400ff9640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 : miss 2026-03-20T11:46:56.141 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa400ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa400ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa400ff9640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 info.flags=0x17 2026-03-20T11:46:56.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa400ff9640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 to cache LRU end 2026-03-20T11:46:56.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa400ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa400ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa400ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:56.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa400ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-52 : miss 2026-03-20T11:46:56.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa400ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa400ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa400ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-52 info.flags=0x0 2026-03-20T11:46:56.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa400ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-52 to cache LRU end 2026-03-20T11:46:56.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa400ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-52 : miss 2026-03-20T11:46:56.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.144+0000 7fa400ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa400ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa400ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-52 info.flags=0x0 2026-03-20T11:46:56.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa400ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-52 to cache LRU end 2026-03-20T11:46:56.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa400ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12): adding source pipe: {s={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa400ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12): adding dest pipe: {s={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa400ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa400ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa400ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:56.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d073dae30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d073dae30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d073dae30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d073dae30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07873360:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078ab7b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07873360:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 20 run: stack=0x558d0789c890 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078a9fd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078a9fd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = f67cd3dc38bb6d37f2b9f4144e7cc1ecc576a3c21b76cc70547d97f71cd4ff75 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:f67cd3dc38bb6d37f2b9f4144e7cc1ecc576a3c21b76cc70547d97f71cd4ff75 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.143 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = cf982ecd36674775c27b9ee3af507ff03535c49cc5dcd35f135418205f2b1cb3 2026-03-20T11:46:56.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=cf982ecd36674775c27b9ee3af507ff03535c49cc5dcd35f135418205f2b1cb3 2026-03-20T11:46:56.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=12, curl_handle=0x558d0786c860 2026-03-20T11:46:56.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa44df93900 20 run: stack=0x558d0786aaf0 is io blocked 2026-03-20T11:46:56.144 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.145+0000 7fa3f0ff9640 20 link_request req_data=0x558d07857470 req_data->id=12, curl_handle=0x558d0786c860 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d078a1e20:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 15 stack 0x558d0785c780 end 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 20 run: stack=0x558d0785c780 is done 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 20 collect(): s=0x558d07856e80 stack=0x558d0785c780 is complete 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 10 trimming bilogs for bucket=:dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) markers=[,00000000001.98.6,,,,00000000001.102.6,,,00000000001.40.6,,00000000001.48.6], shards=11 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0786a4d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 10 trimming bilog shard 1 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.98.6 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 10 trimming bilog shard 5 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.102.6 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 10 trimming bilog shard 8 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.40.6 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 10 trimming bilog shard 10 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.48.6 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 20 run: stack=0x558d07856e80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d078ad160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d0786ac50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d0785e380:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d07870f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d078ad160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.1 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 20 run: stack=0x558d0785c780 is io blocked 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d0786ac50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.5 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is io blocked 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d0785e380:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.8 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 20 run: stack=0x558d07855790 is io blocked 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d07870f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.145 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.10 2026-03-20T11:46:56.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.148+0000 7fa44df93900 20 run: stack=0x558d07876860 is io blocked 2026-03-20T11:46:56.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.149+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d078ad160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.149+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d078ad160:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.149+0000 7fa44df93900 15 stack 0x558d0785c780 end 2026-03-20T11:46:56.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.149+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.149+0000 7fa44df93900 20 run: stack=0x558d0785c780 is done 2026-03-20T11:46:56.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.149+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0786a4d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.149+0000 7fa44df93900 20 run: stack=0x558d07856e80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.149+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d0786ac50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.149+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d0786ac50:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.149+0000 7fa44df93900 15 stack 0x558d0786bfc0 end 2026-03-20T11:46:56.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.149+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.149+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is done 2026-03-20T11:46:56.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.149+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0786a4d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.146 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.149+0000 7fa44df93900 20 run: stack=0x558d07856e80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.149+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d0785e380:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.149+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d0785e380:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.149+0000 7fa44df93900 15 stack 0x558d07855790 end 2026-03-20T11:46:56.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.149+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.149+0000 7fa44df93900 20 run: stack=0x558d07855790 is done 2026-03-20T11:46:56.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.149+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0786a4d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.149+0000 7fa44df93900 20 run: stack=0x558d07856e80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.147 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d073d49b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d073d49b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078a4c10:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 15 stack 0x558d07372f40 end 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 run: stack=0x558d07372f40 is done 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 collect(): s=0x558d0785c1a0 stack=0x558d07372f40 is complete 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 15 stack 0x558d0785c1a0 end 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 stack->operate() returned ret=-22 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is done 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 4 starting trim on bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d07870f10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d07870f10:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 15 stack 0x558d07876860 end 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 run: stack=0x558d07876860 is done 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d0786ff50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0786a4d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d0786ff50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 enqueued request req=0x558d07855aa0 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 req: 0x558d07855aa0 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is io blocked 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 trim: trimmed bucket instance dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 15 stack 0x558d07856e80 end 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 run: stack=0x558d07856e80 is done 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 4 starting trim on bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078751c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078751c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa4007f8640 20 dequeued request req=0x558d07855aa0 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa4007f8640 20 RGWWQ: empty 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa4007f8640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 : miss 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa4007f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 enqueued request req=0x558d0785d060 2026-03-20T11:46:56.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 req: 0x558d0785d060 2026-03-20T11:46:56.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa44df93900 20 run: stack=0x558d07856e80 is io blocked 2026-03-20T11:46:56.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa3ffff7640 20 dequeued request req=0x558d0785d060 2026-03-20T11:46:56.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa3ffff7640 20 RGWWQ: empty 2026-03-20T11:46:56.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa3ffff7640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 : miss 2026-03-20T11:46:56.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa3ffff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa4007f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:56.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa4007f8640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 info.flags=0x17 2026-03-20T11:46:56.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa4007f8640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 to cache LRU end 2026-03-20T11:46:56.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa4007f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa4007f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa4007f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:46:56.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa4007f8640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-42 : miss 2026-03-20T11:46:56.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa4007f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa3ffff7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:46:56.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa3ffff7640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 info.flags=0x17 2026-03-20T11:46:56.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa3ffff7640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 to cache LRU end 2026-03-20T11:46:56.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa3ffff7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa3ffff7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa3ffff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:46:56.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa3ffff7640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-2 : miss 2026-03-20T11:46:56.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa3ffff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa4007f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa4007f8640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-42 info.flags=0x0 2026-03-20T11:46:56.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa4007f8640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-42 to cache LRU end 2026-03-20T11:46:56.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa4007f8640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-42 : miss 2026-03-20T11:46:56.149 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.150+0000 7fa4007f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa3ffff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa3ffff7640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-2 info.flags=0x0 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa3ffff7640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-2 to cache LRU end 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa3ffff7640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-2 : miss 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa3ffff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa4007f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa4007f8640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-42 info.flags=0x0 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa4007f8640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-42 to cache LRU end 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa4007f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2): adding source pipe: {s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa4007f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2): adding dest pipe: {s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa4007f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa4007f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa4007f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d0786ff50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d0786ff50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d0786ff50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d0786ff50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078a4c10:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa3ffff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa3ffff7640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-2 info.flags=0x0 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa3ffff7640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-2 to cache LRU end 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa3ffff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2): adding source pipe: {s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.150 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa3ffff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2): adding dest pipe: {s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 120fa70a38f175e57126d86df5fb541215eebdcc164a7cb74ff4db3bc13ed978 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:120fa70a38f175e57126d86df5fb541215eebdcc164a7cb74ff4db3bc13ed978 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa3ffff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa3ffff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa3ffff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 15e874249a3c9d5809ab3a1ae9de31a0208ca185c291de9a974dc6846b627d12 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=15e874249a3c9d5809ab3a1ae9de31a0208ca185c291de9a974dc6846b627d12 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=13, curl_handle=0x558d0785fe10 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 run: stack=0x558d07876860 is io blocked 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078751c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078751c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078751c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078751c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078a07e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 run: stack=0x558d07856e80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078a1d80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078a1d80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.151 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 1788fc5391d829473dca311690d0c25cba95fc148e60a5937a638caff39071b4 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:1788fc5391d829473dca311690d0c25cba95fc148e60a5937a638caff39071b4 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = b338c39f6b35caf699105cfc45879ee345a8a6703c0c7c6b519e7185bdeaafea 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b338c39f6b35caf699105cfc45879ee345a8a6703c0c7c6b519e7185bdeaafea 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=14, curl_handle=0x558d07868640 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa44df93900 20 run: stack=0x558d07372f40 is io blocked 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa3f0ff9640 20 link_request req_data=0x558d07871cd0 req_data->id=13, curl_handle=0x558d0785fe10 2026-03-20T11:46:56.152 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.151+0000 7fa3f0ff9640 20 link_request req_data=0x558d078a2a40 req_data->id=14, curl_handle=0x558d07868640 2026-03-20T11:46:56.152 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:56.152+0000 7f11f98e3640 -1 req 5910663667989660101 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:56.152 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:56.152+0000 7f123595b640 -1 req 5660695014680234093 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:56.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.155+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d0786c880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.155+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d0786c880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.155+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d0786c880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.155+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d0786c880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.155+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07872a40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.155+0000 7fa44df93900 15 stack 0x558d078a1890 end 2026-03-20T11:46:56.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.155+0000 7fa44df93900 20 run: stack=0x558d078a1890 is done 2026-03-20T11:46:56.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.155+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078541f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.155+0000 7fa44df93900 20 collect(): s=0x558d07878200 stack=0x558d078a1890 is complete 2026-03-20T11:46:56.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.155+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.155+0000 7fa44df93900 10 trimming bilogs for bucket=:dipepu-20[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25]) markers=[,,,,,,,,00000000001.41.14,,], shards=11 2026-03-20T11:46:56.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.155+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0786a4d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.155+0000 7fa44df93900 10 trimming bilog shard 8 of :dipepu-20[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25]) at marker 00000000001.41.14 2026-03-20T11:46:56.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.155+0000 7fa44df93900 20 run: stack=0x558d07878200 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.155+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d073d4a90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.155+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d073d4a90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.155+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25.8 2026-03-20T11:46:56.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.155+0000 7fa44df93900 20 run: stack=0x558d078a1890 is io blocked 2026-03-20T11:46:56.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.156+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d073d4a90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.156+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d073d4a90:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.156+0000 7fa44df93900 15 stack 0x558d078a1890 end 2026-03-20T11:46:56.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.156+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.156+0000 7fa44df93900 20 run: stack=0x558d078a1890 is done 2026-03-20T11:46:56.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.156+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0786a4d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.156+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078541f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.156+0000 7fa44df93900 20 trim: trimmed bucket instance dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:46:56.153 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.156+0000 7fa44df93900 15 stack 0x558d07878200 end 2026-03-20T11:46:56.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.156+0000 7fa44df93900 20 run: stack=0x558d07878200 is done 2026-03-20T11:46:56.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.156+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.156+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.156+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078541f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.156+0000 7fa44df93900 4 starting trim on bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:56.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.156+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0786ff50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.156+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0786ff50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.156+0000 7fa44df93900 20 enqueued request req=0x558d07875c10 2026-03-20T11:46:56.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.156+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.156+0000 7fa44df93900 20 req: 0x558d07875c10 2026-03-20T11:46:56.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.156+0000 7fa44df93900 20 run: stack=0x558d07878200 is io blocked 2026-03-20T11:46:56.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.156+0000 7fa3ff7f6640 20 dequeued request req=0x558d07875c10 2026-03-20T11:46:56.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.156+0000 7fa3ff7f6640 20 RGWWQ: empty 2026-03-20T11:46:56.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.156+0000 7fa3ff7f6640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 : miss 2026-03-20T11:46:56.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.156+0000 7fa3ff7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa3ff7f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa3ff7f6640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 info.flags=0x17 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa3ff7f6640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 to cache LRU end 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa3ff7f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa3ff7f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa3ff7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa3ff7f6640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-57 : miss 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa3ff7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa3ff7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa3ff7f6640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-57 info.flags=0x0 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa3ff7f6640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-57 to cache LRU end 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa3ff7f6640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-57 : miss 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa3ff7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa3ff7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa3ff7f6640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-57 info.flags=0x0 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa3ff7f6640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-57 to cache LRU end 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa3ff7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17): adding source pipe: {s={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa3ff7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17): adding dest pipe: {s={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa3ff7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa3ff7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa3ff7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0786ff50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0786ff50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0786ff50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d0786ff50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078541f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07872a40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078541f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 20 run: stack=0x558d07878200 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d0786c880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d0786c880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.155 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 4409d83a1b722f8433edc939e22c2533c9ef9c3ff69f12195c8ce65feefcab7f 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr:4409d83a1b722f8433edc939e22c2533c9ef9c3ff69f12195c8ce65feefcab7f 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 132fe5cd3032f83fd7b85b7b268aa4fba3c561a1bc5ec4afc336c955eb7dff31 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=132fe5cd3032f83fd7b85b7b268aa4fba3c561a1bc5ec4afc336c955eb7dff31 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=15, curl_handle=0x558d07864550 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.157+0000 7fa44df93900 20 run: stack=0x558d078a1890 is io blocked 2026-03-20T11:46:56.156 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.158+0000 7fa3f0ff9640 20 link_request req_data=0x558d07868b70 req_data->id=15, curl_handle=0x558d07864550 2026-03-20T11:46:56.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078a9fd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078a9fd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078a9fd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078a9fd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078ab7b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 15 stack 0x558d0786aaf0 end 2026-03-20T11:46:56.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 20 run: stack=0x558d0786aaf0 is done 2026-03-20T11:46:56.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07873360:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 20 collect(): s=0x558d0789c890 stack=0x558d0786aaf0 is complete 2026-03-20T11:46:56.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 10 trimming bilogs for bucket=:dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) markers=[00000000002.30.6,,00000000002.60.6,,00000000002.47.6,,,00000000002.30.6,,,], shards=11 2026-03-20T11:46:56.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d0786a4d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 10 trimming bilog shard 0 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.30.6 2026-03-20T11:46:56.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 10 trimming bilog shard 2 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.60.6 2026-03-20T11:46:56.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 10 trimming bilog shard 4 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.47.6 2026-03-20T11:46:56.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 10 trimming bilog shard 7 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.30.6 2026-03-20T11:46:56.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 20 run: stack=0x558d0789c890 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d07858b80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d073dae10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07857c30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d0785a130:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d07858b80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.0 2026-03-20T11:46:56.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 20 run: stack=0x558d0786aaf0 is io blocked 2026-03-20T11:46:56.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d073dae10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.2 2026-03-20T11:46:56.185 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 20 run: stack=0x558d07855790 is io blocked 2026-03-20T11:46:56.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07857c30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.4 2026-03-20T11:46:56.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is io blocked 2026-03-20T11:46:56.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d0785a130:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.7 2026-03-20T11:46:56.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.188+0000 7fa44df93900 20 run: stack=0x558d0785c780 is io blocked 2026-03-20T11:46:56.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.189+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d0785a130:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.189+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d0785a130:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.189+0000 7fa44df93900 15 stack 0x558d0785c780 end 2026-03-20T11:46:56.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.189+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.189+0000 7fa44df93900 20 run: stack=0x558d0785c780 is done 2026-03-20T11:46:56.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.189+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d0786a4d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.186 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.189+0000 7fa44df93900 20 run: stack=0x558d0789c890 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d07858b80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d07858b80:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 15 stack 0x558d0786aaf0 end 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 run: stack=0x558d0786aaf0 is done 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d0786a4d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 run: stack=0x558d0789c890 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07857c30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07857c30:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 15 stack 0x558d0786bfc0 end 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is done 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d0786a4d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 run: stack=0x558d0789c890 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d073dae10:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d073dae10:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 15 stack 0x558d07855790 end 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 run: stack=0x558d07855790 is done 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d0786a4d0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07873360:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 trim: trimmed bucket instance dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 15 stack 0x558d0789c890 end 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 run: stack=0x558d0789c890 is done 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07873360:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 4 starting trim on bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:46:56.187 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d0786ff50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d0786ff50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 enqueued request req=0x558d07876670 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 req: 0x558d07876670 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa44df93900 20 run: stack=0x558d0789c890 is io blocked 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa3feff5640 20 dequeued request req=0x558d07876670 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa3feff5640 20 RGWWQ: empty 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa3feff5640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 : miss 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.190+0000 7fa3feff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa3feff5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa3feff5640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 info.flags=0x17 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa3feff5640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 to cache LRU end 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa3feff5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa3feff5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa3feff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa3feff5640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-29 : miss 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa3feff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa3feff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa3feff5640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-29 info.flags=0x0 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa3feff5640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-29 to cache LRU end 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa3feff5640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-29 : miss 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa3feff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa3feff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa3feff5640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-29 info.flags=0x0 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa3feff5640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-29 to cache LRU end 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa3feff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34): adding source pipe: {s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa3feff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34): adding dest pipe: {s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa3feff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa3feff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa3feff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d0786ff50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.188 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d0786ff50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d0786ff50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d0786ff50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07873360:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d073dc110:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07873360:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa44df93900 20 run: stack=0x558d0789c890 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d078a4460:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d078a4460:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.191+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = f9d1a26f6f93db886650885565c5602bc3a89c2429b371a9a1f9d817d890f5d5 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.192+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:f9d1a26f6f93db886650885565c5602bc3a89c2429b371a9a1f9d817d890f5d5 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.192+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.192+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.192+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.192+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.192+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 23e2b5d40d2cfcd997ae88b0973b161eb050d932142ee2c8e17471877bd54873 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.192+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=23e2b5d40d2cfcd997ae88b0973b161eb050d932142ee2c8e17471877bd54873 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.192+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.192+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.192+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.192+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=16, curl_handle=0x558d0786c860 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.192+0000 7fa44df93900 20 run: stack=0x558d07855790 is io blocked 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.192+0000 7fa3f0ff9640 20 link_request req_data=0x558d07857470 req_data->id=16, curl_handle=0x558d0786c860 2026-03-20T11:46:56.189 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:56.192+0000 7f12ac248640 -1 req 8995270042766928702 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:56.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078a4c10:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 15 stack 0x558d07876860 end 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 run: stack=0x558d07876860 is done 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 collect(): s=0x558d0785c1a0 stack=0x558d07876860 is complete 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 15 stack 0x558d0785c1a0 end 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 stack->operate() returned ret=-22 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is done 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 4 starting trim on bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a6060:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a6060:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 enqueued request req=0x558d07871cd0 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 req: 0x558d07871cd0 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is io blocked 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa3fe7f4640 20 dequeued request req=0x558d07871cd0 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa3fe7f4640 20 RGWWQ: empty 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa3fe7f4640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 : miss 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa3fe7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa3fe7f4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa3fe7f4640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 info.flags=0x17 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa3fe7f4640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 to cache LRU end 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa3fe7f4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa3fe7f4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa3fe7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa3fe7f4640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-23 : miss 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa3fe7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa3fe7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa3fe7f4640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-23 info.flags=0x0 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa3fe7f4640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-23 to cache LRU end 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa3fe7f4640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-23 : miss 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa3fe7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa3fe7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa3fe7f4640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-23 info.flags=0x0 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa3fe7f4640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-23 to cache LRU end 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa3fe7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28): adding source pipe: {s={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.190 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa3fe7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28): adding dest pipe: {s={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa3fe7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa3fe7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa3fe7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:56.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a6060:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a6060:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a6060:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a6060:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078a4c10:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.193+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.191 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.191 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.191 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.191 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.191 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 0546d37394c10f81c22fc54715cf2c2759b8336e25a77e92618dba71a280a8ba 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:0546d37394c10f81c22fc54715cf2c2759b8336e25a77e92618dba71a280a8ba 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 98b1ad297ba16fa37083c69ca3794f151bd8ba7ae73055c03b679d814cf21c26 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=98b1ad297ba16fa37083c69ca3794f151bd8ba7ae73055c03b679d814cf21c26 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=17, curl_handle=0x558d07868640 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 20 run: stack=0x558d07876860 is io blocked 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078a1d80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078a1d80:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa3f0ff9640 20 link_request req_data=0x558d078522e0 req_data->id=17, curl_handle=0x558d07868640 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078a07e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 15 stack 0x558d07372f40 end 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 20 run: stack=0x558d07372f40 is done 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 20 collect(): s=0x558d07856e80 stack=0x558d07372f40 is complete 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 15 stack 0x558d07856e80 end 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 20 stack->operate() returned ret=-22 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 20 run: stack=0x558d07856e80 is done 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 4 starting trim on bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a6060:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a6060:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 20 enqueued request req=0x558d0785d060 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 20 req: 0x558d0785d060 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa44df93900 20 run: stack=0x558d07856e80 is io blocked 2026-03-20T11:46:56.192 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa3fdff3640 20 dequeued request req=0x558d0785d060 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa3fdff3640 20 RGWWQ: empty 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa3fdff3640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 : miss 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa3fdff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa3fdff3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa3fdff3640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 info.flags=0x17 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa3fdff3640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 to cache LRU end 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa3fdff3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa3fdff3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa3fdff3640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa3fdff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa3fdff3640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-34 : miss 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa3fdff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa3fdff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa3fdff3640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-34 info.flags=0x0 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa3fdff3640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-34 to cache LRU end 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa3fdff3640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-34 : miss 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.194+0000 7fa3fdff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa3fdff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa3fdff3640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-34 info.flags=0x0 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa3fdff3640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-34 to cache LRU end 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa3fdff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39): adding source pipe: {s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa3fdff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39): adding dest pipe: {s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa3fdff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa3fdff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa3fdff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a6060:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a6060:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a6060:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a6060:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078a07e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 20 run: stack=0x558d07856e80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078a1d80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078a1d80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.193 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.194 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.194 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.194 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 726ae64fc6681090a5477ae19071b0ea05186f4b9c8a0b34ee63f8f8b65aa9c0 2026-03-20T11:46:56.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.194 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.194 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.194 INFO:teuthology.orchestra.run.vm02.stderr:726ae64fc6681090a5477ae19071b0ea05186f4b9c8a0b34ee63f8f8b65aa9c0 2026-03-20T11:46:56.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 99d712d960f9f1c5de4a3b6e671c2636f85ba86b8d1f35fe813a5c264c66cfcc 2026-03-20T11:46:56.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=99d712d960f9f1c5de4a3b6e671c2636f85ba86b8d1f35fe813a5c264c66cfcc 2026-03-20T11:46:56.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=18, curl_handle=0x558d0785fe10 2026-03-20T11:46:56.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa44df93900 20 run: stack=0x558d07372f40 is io blocked 2026-03-20T11:46:56.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.195+0000 7fa3f0ff9640 20 link_request req_data=0x558d078a2a40 req_data->id=18, curl_handle=0x558d0785fe10 2026-03-20T11:46:56.194 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:56.195+0000 7f12d2a95640 -1 req 2452489921343078709 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:56.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.201+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d0786c880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.201+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d0786c880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.201+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d0786c880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.201+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d0786c880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.201+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07872a40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.201+0000 7fa44df93900 15 stack 0x558d078a1890 end 2026-03-20T11:46:56.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.201+0000 7fa44df93900 20 run: stack=0x558d078a1890 is done 2026-03-20T11:46:56.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.201+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078541f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.201+0000 7fa44df93900 20 collect(): s=0x558d07878200 stack=0x558d078a1890 is complete 2026-03-20T11:46:56.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.201+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.201+0000 7fa44df93900 10 trimming bilogs for bucket=:dipepu-57[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17]) markers=[,00000000001.63.6,,,,00000000001.43.6,,,,,], shards=11 2026-03-20T11:46:56.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.201+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078a6820:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.201+0000 7fa44df93900 10 trimming bilog shard 1 of :dipepu-57[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17]) at marker 00000000001.63.6 2026-03-20T11:46:56.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.201+0000 7fa44df93900 10 trimming bilog shard 5 of :dipepu-57[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17]) at marker 00000000001.43.6 2026-03-20T11:46:56.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.201+0000 7fa44df93900 20 run: stack=0x558d07878200 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.201+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d073d9f80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.201+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07667780:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.201+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d073d9f80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.201+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17.1 2026-03-20T11:46:56.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.201+0000 7fa44df93900 20 run: stack=0x558d078a1890 is io blocked 2026-03-20T11:46:56.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.201+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07667780:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.201+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17.5 2026-03-20T11:46:56.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.201+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is io blocked 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07667780:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07667780:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa44df93900 15 stack 0x558d0786bfc0 end 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is done 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078a6820:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa44df93900 20 run: stack=0x558d07878200 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d073d9f80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d073d9f80:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa44df93900 15 stack 0x558d078a1890 end 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa44df93900 20 run: stack=0x558d078a1890 is done 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078a6820:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078541f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa44df93900 20 trim: trimmed bucket instance dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa44df93900 15 stack 0x558d07878200 end 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa44df93900 20 run: stack=0x558d07878200 is done 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078c0e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa44df93900 4 starting trim on bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d07665ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d07665ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa44df93900 20 enqueued request req=0x558d07875c10 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa44df93900 20 req: 0x558d07875c10 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa44df93900 20 run: stack=0x558d07878200 is io blocked 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa3fd7f2640 20 dequeued request req=0x558d07875c10 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa3fd7f2640 20 RGWWQ: empty 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa3fd7f2640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 : miss 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa3fd7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa3fd7f2640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa3fd7f2640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 info.flags=0x17 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa3fd7f2640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 to cache LRU end 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa3fd7f2640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa3fd7f2640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa3fd7f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa3fd7f2640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-54 : miss 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.202+0000 7fa3fd7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa3fd7f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa3fd7f2640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-54 info.flags=0x0 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa3fd7f2640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-54 to cache LRU end 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa3fd7f2640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-54 : miss 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa3fd7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa3fd7f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa3fd7f2640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-54 info.flags=0x0 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa3fd7f2640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-54 to cache LRU end 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa3fd7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14): adding source pipe: {s={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa3fd7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14): adding dest pipe: {s={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa3fd7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa3fd7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa3fd7f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d07665ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d07665ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d07665ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d07665ca0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078c0e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07872a40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078c0e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 20 run: stack=0x558d07878200 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d0786c880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d0786c880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.200 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.201 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.201 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.201 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.201 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.201 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.201 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.201 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.201 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.201 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 3fd1a386862e0c3fa0e7ad1dc6232d08bbbebd677392de145e06b67e14825248 2026-03-20T11:46:56.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.201 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.201 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.201 INFO:teuthology.orchestra.run.vm02.stderr:3fd1a386862e0c3fa0e7ad1dc6232d08bbbebd677392de145e06b67e14825248 2026-03-20T11:46:56.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = c2c9c7e44df13a753fc9b520c98f047f6cf9ed599f55ad19032c05de80eb6ee1 2026-03-20T11:46:56.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c2c9c7e44df13a753fc9b520c98f047f6cf9ed599f55ad19032c05de80eb6ee1 2026-03-20T11:46:56.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=19, curl_handle=0x558d07864550 2026-03-20T11:46:56.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa44df93900 20 run: stack=0x558d078a1890 is io blocked 2026-03-20T11:46:56.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.203+0000 7fa3f0ff9640 20 link_request req_data=0x558d07868b70 req_data->id=19, curl_handle=0x558d07864550 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d078a4460:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d078a4460:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d073dc110:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 15 stack 0x558d07855790 end 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 run: stack=0x558d07855790 is done 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07873360:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 collect(): s=0x558d0789c890 stack=0x558d07855790 is complete 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07873360:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 15 stack 0x558d0789c890 end 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 stack->operate() returned ret=-22 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 run: stack=0x558d0789c890 is done 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078541f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 4 starting trim on bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d073d9f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d073d9f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 enqueued request req=0x558d07876670 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 req: 0x558d07876670 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 run: stack=0x558d0789c890 is io blocked 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa3fcff1640 20 dequeued request req=0x558d07876670 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa3fcff1640 20 RGWWQ: empty 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa3fcff1640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 : miss 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa3fcff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa3fcff1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.230 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa3fcff1640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 info.flags=0x17 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa3fcff1640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 to cache LRU end 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa3fcff1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa3fcff1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa3fcff1640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa3fcff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa3fcff1640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-32 : miss 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa3fcff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa3fcff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa3fcff1640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-32 info.flags=0x0 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa3fcff1640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-32 to cache LRU end 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa3fcff1640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-32 : miss 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa3fcff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa3fcff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa3fcff1640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-32 info.flags=0x0 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa3fcff1640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-32 to cache LRU end 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa3fcff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37): adding source pipe: {s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa3fcff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37): adding dest pipe: {s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa3fcff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa3fcff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa3fcff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d073d9f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d073d9f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d073d9f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d073d9f80:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078541f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d076626a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078541f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 run: stack=0x558d0789c890 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d078a4410:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d078a4410:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.233+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.234+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.234+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.234+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.234+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 874cf20e0efe4057b67669cded6b06b1adae431323eb5ec2c34a038829e6da2b 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.234+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:874cf20e0efe4057b67669cded6b06b1adae431323eb5ec2c34a038829e6da2b 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.234+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.234+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.234+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.234+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.231 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.234+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 7b9e0302ac06068fee0c2ebc65e9eb8c80ec13f0e2159edae0561846c98468c3 2026-03-20T11:46:56.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.234+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=7b9e0302ac06068fee0c2ebc65e9eb8c80ec13f0e2159edae0561846c98468c3 2026-03-20T11:46:56.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.234+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.234+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.234+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.232 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:56.234+0000 7f128fa0f640 -1 req 10327723434055154463 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:56.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.234+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=20, curl_handle=0x558d0786c860 2026-03-20T11:46:56.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.234+0000 7fa44df93900 20 run: stack=0x558d07855790 is io blocked 2026-03-20T11:46:56.232 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.234+0000 7fa3f0ff9640 20 link_request req_data=0x558d07857470 req_data->id=20, curl_handle=0x558d0786c860 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078a1d80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078a1d80:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078a07e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 15 stack 0x558d07372f40 end 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 run: stack=0x558d07372f40 is done 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 collect(): s=0x558d07856e80 stack=0x558d07372f40 is complete 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 15 stack 0x558d07856e80 end 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 stack->operate() returned ret=-22 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 run: stack=0x558d07856e80 is done 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078a4c10:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 15 stack 0x558d07876860 end 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 run: stack=0x558d07876860 is done 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 collect(): s=0x558d0785c1a0 stack=0x558d07876860 is complete 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 10 trimming bilogs for bucket=:dipepu-23[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28]) markers=[,,,,,,,,00000000002.27.13,,], shards=11 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 4 starting trim on bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a4c10:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 10 trimming bilog shard 8 of :dipepu-23[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28]) at marker 00000000002.27.13 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0786ac50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d0785a130:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0786ac50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 enqueued request req=0x558d078ad9b0 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 req: 0x558d078ad9b0 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 run: stack=0x558d07856e80 is io blocked 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d0785a130:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28.8 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa44df93900 20 run: stack=0x558d07876860 is io blocked 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa3fc7f0640 20 dequeued request req=0x558d078ad9b0 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa3fc7f0640 20 RGWWQ: empty 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa3fc7f0640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 : miss 2026-03-20T11:46:56.233 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa3fc7f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa3fc7f0640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa3fc7f0640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 info.flags=0x17 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa3fc7f0640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 to cache LRU end 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa3fc7f0640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa3fc7f0640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa3fc7f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa3fc7f0640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-19 : miss 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa3fc7f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa3fc7f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa3fc7f0640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-19 info.flags=0x0 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa3fc7f0640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-19 to cache LRU end 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa3fc7f0640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-19 : miss 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.236+0000 7fa3fc7f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa3fc7f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa3fc7f0640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-19 info.flags=0x0 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa3fc7f0640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-19 to cache LRU end 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa3fc7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24): adding source pipe: {s={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa3fc7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24): adding dest pipe: {s={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa3fc7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa3fc7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa3fc7f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0786ac50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0786ac50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0786ac50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d0786ac50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078a07e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 run: stack=0x558d07856e80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078a1d80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078a1d80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.234 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 4d1a2c430d8e5524a2a086b2820d9b4dfb80ebbce9639c0858385e961f707da5 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:4d1a2c430d8e5524a2a086b2820d9b4dfb80ebbce9639c0858385e961f707da5 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = a06296dd9a505aca59678e9c650f8bdc74dedebe308c95980121d212f0c742bc 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a06296dd9a505aca59678e9c650f8bdc74dedebe308c95980121d212f0c742bc 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=21, curl_handle=0x558d0785fe10 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 run: stack=0x558d07372f40 is io blocked 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa3f0ff9640 20 link_request req_data=0x558d078a2a40 req_data->id=21, curl_handle=0x558d0785fe10 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d0785a130:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d0785a130:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 15 stack 0x558d07876860 end 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 run: stack=0x558d07876860 is done 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a4c10:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 trim: trimmed bucket instance dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 15 stack 0x558d0785c1a0 end 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is done 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 4 starting trim on bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d0785a130:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d0785a130:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 enqueued request req=0x558d078a3c40 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.235 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 req: 0x558d078a3c40 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa3fbfef640 20 dequeued request req=0x558d078a3c40 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa3fbfef640 20 RGWWQ: empty 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa3fbfef640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 : miss 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa3fbfef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.237+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is io blocked 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa3fbfef640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa3fbfef640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 info.flags=0x17 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa3fbfef640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 to cache LRU end 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa3fbfef640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa3fbfef640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa3fbfef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa3fbfef640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-4 : miss 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa3fbfef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa3fbfef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa3fbfef640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-4 info.flags=0x0 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa3fbfef640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-4 to cache LRU end 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa3fbfef640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-4 : miss 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa3fbfef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa3fbfef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa3fbfef640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-4 info.flags=0x0 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa3fbfef640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-4 to cache LRU end 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa3fbfef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5): adding source pipe: {s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa3fbfef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5): adding dest pipe: {s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa3fbfef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa3fbfef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa3fbfef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d0785a130:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d0785a130:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d0785a130:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d0785a130:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078a4c10:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.236 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.237 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.237 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.237 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.237 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.237 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.237 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 4e2a6d003e81f1e0c7261d27d3696b07ee59632eeccdd490e845e74a2fe17f37 2026-03-20T11:46:56.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.237 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.237 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.237 INFO:teuthology.orchestra.run.vm02.stderr:4e2a6d003e81f1e0c7261d27d3696b07ee59632eeccdd490e845e74a2fe17f37 2026-03-20T11:46:56.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 36c820daf9f749cec92e23765cc0fc487fc7274f00089aad343ff55ecf90f584 2026-03-20T11:46:56.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=36c820daf9f749cec92e23765cc0fc487fc7274f00089aad343ff55ecf90f584 2026-03-20T11:46:56.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=22, curl_handle=0x558d07868640 2026-03-20T11:46:56.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa44df93900 20 run: stack=0x558d07876860 is io blocked 2026-03-20T11:46:56.237 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.238+0000 7fa3f0ff9640 20 link_request req_data=0x558d078522e0 req_data->id=22, curl_handle=0x558d07868640 2026-03-20T11:46:56.237 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:56.239+0000 7f12018f3640 -1 req 878153759269107536 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:56.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d0786c880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d0786c880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d0786c880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d0786c880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07872a40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 15 stack 0x558d078a1890 end 2026-03-20T11:46:56.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 run: stack=0x558d078a1890 is done 2026-03-20T11:46:56.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078c0e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 collect(): s=0x558d07878200 stack=0x558d078a1890 is complete 2026-03-20T11:46:56.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 10 trimming bilogs for bucket=:dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) markers=[00000000004.65.1,00000000002.57.1,00000000004.95.1,00000000002.36.1,00000000004.41.1,00000000002.67.1,00000000002.53.1,00000000004.82.1,00000000002.67.1,00000000002.123.1,00000000002.39.1], shards=11 2026-03-20T11:46:56.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d073dba40:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 10 trimming bilog shard 0 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.65.1 2026-03-20T11:46:56.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 10 trimming bilog shard 1 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.57.1 2026-03-20T11:46:56.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 10 trimming bilog shard 2 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.95.1 2026-03-20T11:46:56.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 10 trimming bilog shard 3 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.36.1 2026-03-20T11:46:56.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 10 trimming bilog shard 4 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.41.1 2026-03-20T11:46:56.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 10 trimming bilog shard 5 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.67.1 2026-03-20T11:46:56.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 10 trimming bilog shard 6 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.53.1 2026-03-20T11:46:56.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 10 trimming bilog shard 7 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.82.1 2026-03-20T11:46:56.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 10 trimming bilog shard 8 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.67.1 2026-03-20T11:46:56.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 10 trimming bilog shard 9 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.123.1 2026-03-20T11:46:56.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 10 trimming bilog shard 10 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.39.1 2026-03-20T11:46:56.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 run: stack=0x558d07878200 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d078c2140:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.243 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d0785a130:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.244 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078c30b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.244 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d07666bb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.244 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d07665a60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.244 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d078a4010:op=0x558d078c5270:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d076613e0:op=0x558d0785e1c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d078ac480:op=0x558d078c6b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d073d8ad0:op=0x558d078c7350:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07662cc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d078aea50:op=0x558d076634c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d078c2140:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.0 2026-03-20T11:46:56.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 run: stack=0x558d078a1890 is io blocked 2026-03-20T11:46:56.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d0785a130:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.1 2026-03-20T11:46:56.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is io blocked 2026-03-20T11:46:56.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078c30b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.2 2026-03-20T11:46:56.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 run: stack=0x558d0786aaf0 is io blocked 2026-03-20T11:46:56.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d07666bb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.3 2026-03-20T11:46:56.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 run: stack=0x558d0785c780 is io blocked 2026-03-20T11:46:56.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d07665a60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.4 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 run: stack=0x558d07860730 is io blocked 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d078a4010:op=0x558d078c5270:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.5 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 run: stack=0x558d078a4010 is io blocked 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d076613e0:op=0x558d0785e1c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.6 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 run: stack=0x558d076613e0 is io blocked 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d078ac480:op=0x558d078c6b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.7 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 run: stack=0x558d078ac480 is io blocked 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d073d8ad0:op=0x558d078c7350:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.8 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 run: stack=0x558d073d8ad0 is io blocked 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07662cc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.9 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 run: stack=0x558d078c7b50 is io blocked 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 cr:s=0x558d078aea50:op=0x558d076634c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.10 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.245+0000 7fa44df93900 20 run: stack=0x558d078aea50 is io blocked 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.246+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d078c2140:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.246+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d078c2140:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.246+0000 7fa44df93900 15 stack 0x558d078a1890 end 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.246+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.246+0000 7fa44df93900 20 run: stack=0x558d078a1890 is done 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.246+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d073dba40:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.246+0000 7fa44df93900 20 run: stack=0x558d07878200 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.246+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07662cc0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.246+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07662cc0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.246+0000 7fa44df93900 15 stack 0x558d078c7b50 end 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.246+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.246+0000 7fa44df93900 20 run: stack=0x558d078c7b50 is done 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.246+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d073dba40:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.246+0000 7fa44df93900 20 run: stack=0x558d07878200 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078c30b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078c30b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 15 stack 0x558d0786aaf0 end 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 run: stack=0x558d0786aaf0 is done 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d073dba40:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 run: stack=0x558d07878200 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 cr:s=0x558d078aea50:op=0x558d076634c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 cr:s=0x558d078aea50:op=0x558d076634c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 15 stack 0x558d078aea50 end 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 run: stack=0x558d078aea50 is done 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d07666bb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d07666bb0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 15 stack 0x558d0785c780 end 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 run: stack=0x558d0785c780 is done 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 cr:s=0x558d076613e0:op=0x558d0785e1c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 cr:s=0x558d076613e0:op=0x558d0785e1c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 15 stack 0x558d076613e0 end 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 run: stack=0x558d076613e0 is done 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d07665a60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d07665a60:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 15 stack 0x558d07860730 end 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 run: stack=0x558d07860730 is done 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 cr:s=0x558d078ac480:op=0x558d078c6b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 cr:s=0x558d078ac480:op=0x558d078c6b50:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 15 stack 0x558d078ac480 end 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 run: stack=0x558d078ac480 is done 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d0785a130:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d0785a130:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 15 stack 0x558d0786bfc0 end 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is done 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d073dba40:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d073dba40:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d073dba40:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d073dba40:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d073dba40:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d073dba40:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.247+0000 7fa44df93900 20 run: stack=0x558d07878200 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa44df93900 20 cr:s=0x558d073d8ad0:op=0x558d078c7350:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa44df93900 20 cr:s=0x558d073d8ad0:op=0x558d078c7350:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa44df93900 15 stack 0x558d073d8ad0 end 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa44df93900 20 run: stack=0x558d073d8ad0 is done 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d073dba40:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa44df93900 20 run: stack=0x558d07878200 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa44df93900 20 cr:s=0x558d078a4010:op=0x558d078c5270:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa44df93900 20 cr:s=0x558d078a4010:op=0x558d078c5270:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa44df93900 15 stack 0x558d078a4010 end 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa44df93900 20 run: stack=0x558d078a4010 is done 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d073dba40:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa44df93900 20 cr:s=0x558d07878200:op=0x558d078c0e70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa44df93900 20 trim: trimmed bucket instance dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa44df93900 15 stack 0x558d07878200 end 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa44df93900 20 run: stack=0x558d07878200 is done 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa44df93900 20 cr:s=0x558d076613e0:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa44df93900 4 starting trim on bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa44df93900 20 cr:s=0x558d076613e0:op=0x558d078af2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa44df93900 20 cr:s=0x558d076613e0:op=0x558d078af2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa44df93900 20 enqueued request req=0x558d07859470 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa44df93900 20 req: 0x558d07859470 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa44df93900 20 run: stack=0x558d076613e0 is io blocked 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa3fb7ee640 20 dequeued request req=0x558d07859470 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa3fb7ee640 20 RGWWQ: empty 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa3fb7ee640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 : miss 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa3fb7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa3fb7ee640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa3fb7ee640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 info.flags=0x17 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa3fb7ee640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 to cache LRU end 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa3fb7ee640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa3fb7ee640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa3fb7ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa3fb7ee640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-53 : miss 2026-03-20T11:46:56.250 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.248+0000 7fa3fb7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa3fb7ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa3fb7ee640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-53 info.flags=0x0 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa3fb7ee640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-53 to cache LRU end 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa3fb7ee640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-53 : miss 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa3fb7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa3fb7ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa3fb7ee640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-53 info.flags=0x0 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa3fb7ee640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-53 to cache LRU end 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa3fb7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13): adding source pipe: {s={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa3fb7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13): adding dest pipe: {s={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa3fb7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa3fb7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa3fb7ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 20 cr:s=0x558d076613e0:op=0x558d078af2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 20 cr:s=0x558d076613e0:op=0x558d078af2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 20 cr:s=0x558d076613e0:op=0x558d078af2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 20 cr:s=0x558d076613e0:op=0x558d078af2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 20 cr:s=0x558d076613e0:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d07873b00:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 20 cr:s=0x558d076613e0:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 20 run: stack=0x558d076613e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d073dcd20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d073dcd20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 730c04baa866fbed1aa8991839fe01eb782ea5a78282f44c10d372c6b0f4a720 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:730c04baa866fbed1aa8991839fe01eb782ea5a78282f44c10d372c6b0f4a720 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 1c49e70e5e0b90a9e970a69d26eb64c79de33436c55151d29c13318aab619039 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=1c49e70e5e0b90a9e970a69d26eb64c79de33436c55151d29c13318aab619039 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=23, curl_handle=0x558d07864550 2026-03-20T11:46:56.251 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa44df93900 20 run: stack=0x558d07860730 is io blocked 2026-03-20T11:46:56.252 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.249+0000 7fa3f0ff9640 20 link_request req_data=0x558d07868b70 req_data->id=23, curl_handle=0x558d07864550 2026-03-20T11:46:56.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d078a4410:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d078a4410:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:56.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d076626a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa44df93900 15 stack 0x558d07855790 end 2026-03-20T11:46:56.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa44df93900 20 run: stack=0x558d07855790 is done 2026-03-20T11:46:56.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078541f0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa44df93900 20 collect(): s=0x558d0789c890 stack=0x558d07855790 is complete 2026-03-20T11:46:56.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa44df93900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:56.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa44df93900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:56.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078541f0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:56.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa44df93900 15 stack 0x558d0789c890 end 2026-03-20T11:46:56.273 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa44df93900 20 stack->operate() returned ret=-22 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa44df93900 20 run: stack=0x558d0789c890 is done 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa44df93900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4e40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa44df93900 4 starting trim on bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078af2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078af2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa44df93900 20 enqueued request req=0x558d07876670 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa44df93900 20 req: 0x558d07876670 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa44df93900 20 run: stack=0x558d0789c890 is io blocked 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa3fafed640 20 dequeued request req=0x558d07876670 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa3fafed640 20 RGWWQ: empty 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa3fafed640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 : miss 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa3fafed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa3fafed640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa3fafed640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 info.flags=0x17 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa3fafed640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 to cache LRU end 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa3fafed640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa3fafed640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa3fafed640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa3fafed640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-14 : miss 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa3fafed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa3fafed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa3fafed640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-14 info.flags=0x0 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa3fafed640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-14 to cache LRU end 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa3fafed640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-14 : miss 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.275+0000 7fa3fafed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa3fafed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa3fafed640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-14 info.flags=0x0 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa3fafed640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-14 to cache LRU end 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa3fafed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19): adding source pipe: {s={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa3fafed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19): adding dest pipe: {s={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa3fafed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa3fafed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa3fafed640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078af2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078af2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078af2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078af2f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4e40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d076626a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4e40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 20 run: stack=0x558d0789c890 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d07858550:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d07858550:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.274 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.275 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.275 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.275 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.275 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.275 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.275 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.275 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.275 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.275 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = cf7edd69407ca9b86174b4cc2b80de599df00e0444f8a5ac53b5088bede92608 2026-03-20T11:46:56.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.275 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.275 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.275 INFO:teuthology.orchestra.run.vm02.stderr:cf7edd69407ca9b86174b4cc2b80de599df00e0444f8a5ac53b5088bede92608 2026-03-20T11:46:56.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = d2440f6260a0e6911307d17cf322e9603b7b63ab9d82ea193bbc01145177e981 2026-03-20T11:46:56.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=d2440f6260a0e6911307d17cf322e9603b7b63ab9d82ea193bbc01145177e981 2026-03-20T11:46:56.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=24, curl_handle=0x558d0786c860 2026-03-20T11:46:56.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa44df93900 20 run: stack=0x558d07855790 is io blocked 2026-03-20T11:46:56.275 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.276+0000 7fa3f0ff9640 20 link_request req_data=0x558d07857470 req_data->id=24, curl_handle=0x558d0786c860 2026-03-20T11:46:56.276 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078a1d80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078a1d80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:56.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078a1d80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078a4c10:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 15 stack 0x558d07876860 end 2026-03-20T11:46:56.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 run: stack=0x558d07876860 is done 2026-03-20T11:46:56.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078a1d80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 collect(): s=0x558d0785c1a0 stack=0x558d07876860 is complete 2026-03-20T11:46:56.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:56.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:56.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:56.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 15 stack 0x558d0785c1a0 end 2026-03-20T11:46:56.277 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 stack->operate() returned ret=-22 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is done 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078a07e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 15 stack 0x558d07372f40 end 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 run: stack=0x558d07372f40 is done 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 collect(): s=0x558d07856e80 stack=0x558d07372f40 is complete 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 10 trimming bilogs for bucket=:dipepu-19[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24]) markers=[,,,,,,,,00000000005.35.12,,], shards=11 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 4 starting trim on bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a07e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 10 trimming bilog shard 8 of :dipepu-19[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24]) at marker 00000000005.35.12 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 run: stack=0x558d07856e80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d07666bb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d073d79b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d07666bb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 enqueued request req=0x558d073db6d0 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 req: 0x558d073db6d0 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is io blocked 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d073d79b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24.8 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa44df93900 20 run: stack=0x558d07372f40 is io blocked 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa3fa7ec640 20 dequeued request req=0x558d073db6d0 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa3fa7ec640 20 RGWWQ: empty 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa3fa7ec640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 : miss 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa3fa7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa3fa7ec640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa3fa7ec640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 info.flags=0x17 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa3fa7ec640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 to cache LRU end 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa3fa7ec640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa3fa7ec640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa3fa7ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa3fa7ec640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-36 : miss 2026-03-20T11:46:56.278 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa3fa7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa3fa7ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa3fa7ec640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-36 info.flags=0x0 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa3fa7ec640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-36 to cache LRU end 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa3fa7ec640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-36 : miss 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.280+0000 7fa3fa7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa3fa7ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa3fa7ec640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-36 info.flags=0x0 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa3fa7ec640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-36 to cache LRU end 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa3fa7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41): adding source pipe: {s={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa3fa7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41): adding dest pipe: {s={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa3fa7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa3fa7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa3fa7ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d07666bb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d07666bb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d07666bb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d07666bb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078a4c10:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078a1d80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078a1d80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = d52e8ec0f8b91a98969d60157c23a0b2441ef8ad209cf483e164b7370a6ec880 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.279 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:d52e8ec0f8b91a98969d60157c23a0b2441ef8ad209cf483e164b7370a6ec880 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = c4621450da8c8eb951790b546c8e5a47c1bad0f96477f5f51766d1de4c4dc55d 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c4621450da8c8eb951790b546c8e5a47c1bad0f96477f5f51766d1de4c4dc55d 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=25, curl_handle=0x558d07868640 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 run: stack=0x558d07876860 is io blocked 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa3f0ff9640 20 link_request req_data=0x558d078522e0 req_data->id=25, curl_handle=0x558d07868640 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d073d79b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d073d79b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 15 stack 0x558d07372f40 end 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 run: stack=0x558d07372f40 is done 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a07e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 trim: trimmed bucket instance dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 15 stack 0x558d07856e80 end 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 run: stack=0x558d07856e80 is done 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 4 starting trim on bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 enqueued request req=0x558d078abc30 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 req: 0x558d078abc30 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa44df93900 20 run: stack=0x558d07856e80 is io blocked 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa3f9feb640 20 dequeued request req=0x558d078abc30 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa3f9feb640 20 RGWWQ: empty 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa3f9feb640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 : miss 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.281+0000 7fa3f9feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa3f9feb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa3f9feb640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 info.flags=0x17 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa3f9feb640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 to cache LRU end 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa3f9feb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa3f9feb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa3f9feb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa3f9feb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-22 : miss 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa3f9feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa3f9feb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.280 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa3f9feb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-22 info.flags=0x0 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa3f9feb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-22 to cache LRU end 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa3f9feb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-22 : miss 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa3f9feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa3f9feb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa3f9feb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-22 info.flags=0x0 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa3f9feb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-22 to cache LRU end 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa3f9feb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27): adding source pipe: {s={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa3f9feb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27): adding dest pipe: {s={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa3f9feb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa3f9feb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa3f9feb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078a07e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa44df93900 20 run: stack=0x558d07856e80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 08aaddb6aa55a1492d681570b6f82eb3c6648b91edb4eb68c3399e869f50a5c7 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.282+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:08aaddb6aa55a1492d681570b6f82eb3c6648b91edb4eb68c3399e869f50a5c7 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.283+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.283+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.283+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.283+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.283+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 417d53562c3aa93f08e10b78f4a5cf436ead625b4640dbcfed51cecd3fb4a9be 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.283+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=417d53562c3aa93f08e10b78f4a5cf436ead625b4640dbcfed51cecd3fb4a9be 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.283+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.283+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.283+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.283+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=26, curl_handle=0x558d0785fe10 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.283+0000 7fa44df93900 20 run: stack=0x558d07372f40 is io blocked 2026-03-20T11:46:56.281 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.283+0000 7fa3f0ff9640 20 link_request req_data=0x558d078a2a40 req_data->id=26, curl_handle=0x558d0785fe10 2026-03-20T11:46:56.287 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d073dcd20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d073dcd20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d073dcd20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d073dcd20:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d07873b00:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 15 stack 0x558d07860730 end 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 run: stack=0x558d07860730 is done 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d076613e0:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 collect(): s=0x558d076613e0 stack=0x558d07860730 is complete 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 10 trimming bilogs for bucket=:dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) markers=[00000000004.87.6,00000000002.126.1,00000000004.75.6,00000000002.34.1,00000000004.89.6,00000000002.87.1,00000000002.128.1,00000000004.94.6,00000000002.37.1,00000000002.73.1,00000000002.61.1], shards=11 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d076613e0:op=0x558d07663f90:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 10 trimming bilog shard 0 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.87.6 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 10 trimming bilog shard 1 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.126.1 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 10 trimming bilog shard 2 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.75.6 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 10 trimming bilog shard 3 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.34.1 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 10 trimming bilog shard 4 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.89.6 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 10 trimming bilog shard 5 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.87.1 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 10 trimming bilog shard 6 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.128.1 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 10 trimming bilog shard 7 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.94.6 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 10 trimming bilog shard 8 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.37.1 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 10 trimming bilog shard 9 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.73.1 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 10 trimming bilog shard 10 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.61.1 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 run: stack=0x558d076613e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d078af210:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d073dc9f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07665a00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d07866610:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d076674e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07873680:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d07664d00:op=0x558d078c1ce0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d0785f4f0:op=0x558d078c24e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d078c1680:op=0x558d078c9950:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d0785a850:op=0x558d078ca150:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d078ca950:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d078af210:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.0 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 run: stack=0x558d07860730 is io blocked 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d073dc9f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.1 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 run: stack=0x558d0785c780 is io blocked 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07665a00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.2 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is io blocked 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d07866610:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.3 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 run: stack=0x558d0786aaf0 is io blocked 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d076674e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.4 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 run: stack=0x558d078c7b50 is io blocked 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07873680:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.5 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 run: stack=0x558d078a1890 is io blocked 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d07664d00:op=0x558d078c1ce0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.6 2026-03-20T11:46:56.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 run: stack=0x558d07664d00 is io blocked 2026-03-20T11:46:56.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d0785f4f0:op=0x558d078c24e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.7 2026-03-20T11:46:56.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 run: stack=0x558d0785f4f0 is io blocked 2026-03-20T11:46:56.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d078c1680:op=0x558d078c9950:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.8 2026-03-20T11:46:56.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 run: stack=0x558d078c1680 is io blocked 2026-03-20T11:46:56.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d0785a850:op=0x558d078ca150:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.9 2026-03-20T11:46:56.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 run: stack=0x558d0785a850 is io blocked 2026-03-20T11:46:56.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d078ca950:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.10 2026-03-20T11:46:56.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.291+0000 7fa44df93900 20 run: stack=0x558d078599d0 is io blocked 2026-03-20T11:46:56.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d078af210:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d078af210:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 15 stack 0x558d07860730 end 2026-03-20T11:46:56.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 run: stack=0x558d07860730 is done 2026-03-20T11:46:56.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 cr:s=0x558d076613e0:op=0x558d07663f90:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 run: stack=0x558d076613e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d073dc9f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d073dc9f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 15 stack 0x558d0785c780 end 2026-03-20T11:46:56.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 run: stack=0x558d0785c780 is done 2026-03-20T11:46:56.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 cr:s=0x558d07664d00:op=0x558d078c1ce0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 cr:s=0x558d07664d00:op=0x558d078c1ce0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 15 stack 0x558d07664d00 end 2026-03-20T11:46:56.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 run: stack=0x558d07664d00 is done 2026-03-20T11:46:56.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d07866610:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d07866610:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 15 stack 0x558d0786aaf0 end 2026-03-20T11:46:56.290 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 run: stack=0x558d0786aaf0 is done 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d078ca950:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d078ca950:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 15 stack 0x558d078599d0 end 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 run: stack=0x558d078599d0 is done 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 cr:s=0x558d076613e0:op=0x558d07663f90:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 cr:s=0x558d076613e0:op=0x558d07663f90:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 cr:s=0x558d076613e0:op=0x558d07663f90:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 cr:s=0x558d076613e0:op=0x558d07663f90:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 run: stack=0x558d076613e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d076674e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d076674e0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 15 stack 0x558d078c7b50 end 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 run: stack=0x558d078c7b50 is done 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 cr:s=0x558d076613e0:op=0x558d07663f90:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 run: stack=0x558d076613e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07873680:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07873680:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 15 stack 0x558d078a1890 end 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 run: stack=0x558d078a1890 is done 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 cr:s=0x558d078c1680:op=0x558d078c9950:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 cr:s=0x558d078c1680:op=0x558d078c9950:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 15 stack 0x558d078c1680 end 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 run: stack=0x558d078c1680 is done 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 cr:s=0x558d076613e0:op=0x558d07663f90:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07665a00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07665a00:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 15 stack 0x558d0786bfc0 end 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is done 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 cr:s=0x558d076613e0:op=0x558d07663f90:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 cr:s=0x558d076613e0:op=0x558d07663f90:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.291 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.293+0000 7fa44df93900 20 run: stack=0x558d076613e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa44df93900 20 cr:s=0x558d0785f4f0:op=0x558d078c24e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa44df93900 20 cr:s=0x558d0785f4f0:op=0x558d078c24e0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa44df93900 15 stack 0x558d0785f4f0 end 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa44df93900 20 run: stack=0x558d0785f4f0 is done 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa44df93900 20 cr:s=0x558d076613e0:op=0x558d07663f90:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa44df93900 20 run: stack=0x558d076613e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa44df93900 20 cr:s=0x558d0785a850:op=0x558d078ca150:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa44df93900 20 cr:s=0x558d0785a850:op=0x558d078ca150:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa44df93900 15 stack 0x558d0785a850 end 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa44df93900 20 run: stack=0x558d0785a850 is done 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa44df93900 20 cr:s=0x558d076613e0:op=0x558d07663f90:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa44df93900 20 cr:s=0x558d076613e0:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa44df93900 20 trim: trimmed bucket instance dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa44df93900 15 stack 0x558d076613e0 end 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa44df93900 20 run: stack=0x558d076613e0 is done 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa44df93900 4 starting trim on bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d078c8680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d078c8680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa44df93900 20 enqueued request req=0x558d07863d20 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa44df93900 20 req: 0x558d07863d20 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa3f97ea640 20 dequeued request req=0x558d07863d20 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa3f97ea640 20 RGWWQ: empty 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa3f97ea640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 : miss 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa3f97ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is io blocked 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa3f97ea640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa3f97ea640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 info.flags=0x17 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa3f97ea640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 to cache LRU end 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa3f97ea640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa3f97ea640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa3f97ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa3f97ea640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-25 : miss 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa3f97ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa3f97ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa3f97ea640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-25 info.flags=0x0 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa3f97ea640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-25 to cache LRU end 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa3f97ea640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-25 : miss 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.294+0000 7fa3f97ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa3f97ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa3f97ea640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-25 info.flags=0x0 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa3f97ea640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-25 to cache LRU end 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa3f97ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30): adding source pipe: {s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa3f97ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30): adding dest pipe: {s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa3f97ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa3f97ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa3f97ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:46:56.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d078c8680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d078c8680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d078c8680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d078c8680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d073d8940:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07855030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07855030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = dac3e8d3eb8afa7038c4eb655bf12de61f71553a41c20e258dd6f0529fe5f28c 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:dac3e8d3eb8afa7038c4eb655bf12de61f71553a41c20e258dd6f0529fe5f28c 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = a899437f9f3c7cf93ccb9313ed74113962f54bb661cc12262aaa654ed2fdceac 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a899437f9f3c7cf93ccb9313ed74113962f54bb661cc12262aaa654ed2fdceac 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=27, curl_handle=0x558d07864550 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa44df93900 20 run: stack=0x558d078a1890 is io blocked 2026-03-20T11:46:56.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.295+0000 7fa3f0ff9640 20 link_request req_data=0x558d07868b70 req_data->id=27, curl_handle=0x558d07864550 2026-03-20T11:46:56.293 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:56.295+0000 7f12f62dc640 -1 req 17665333637883963897 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:56.314 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.318+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d07858550:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.318+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d07858550:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.318+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d07858550:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.318+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d07858550:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.318+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d076626a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.318+0000 7fa44df93900 15 stack 0x558d07855790 end 2026-03-20T11:46:56.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.318+0000 7fa44df93900 20 run: stack=0x558d07855790 is done 2026-03-20T11:46:56.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.318+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4e40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.318+0000 7fa44df93900 20 collect(): s=0x558d0789c890 stack=0x558d07855790 is complete 2026-03-20T11:46:56.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.318+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.318+0000 7fa44df93900 10 trimming bilogs for bucket=:dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) markers=[,,,,,00000000003.36.10,,,00000000011.47.6,00000000003.43.6,], shards=11 2026-03-20T11:46:56.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.318+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c2910:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.318+0000 7fa44df93900 10 trimming bilog shard 5 of :dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) at marker 00000000003.36.10 2026-03-20T11:46:56.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.318+0000 7fa44df93900 10 trimming bilog shard 8 of :dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) at marker 00000000011.47.6 2026-03-20T11:46:56.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.318+0000 7fa44df93900 10 trimming bilog shard 9 of :dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) at marker 00000000003.43.6 2026-03-20T11:46:56.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.318+0000 7fa44df93900 20 run: stack=0x558d0789c890 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.318+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d073d9f80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.318+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d073dcb00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.318+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d0785e260:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.318+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d073d9f80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.318+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19.5 2026-03-20T11:46:56.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.318+0000 7fa44df93900 20 run: stack=0x558d07855790 is io blocked 2026-03-20T11:46:56.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.318+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d073dcb00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.318+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19.8 2026-03-20T11:46:56.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.318+0000 7fa44df93900 20 run: stack=0x558d078c7b50 is io blocked 2026-03-20T11:46:56.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.318+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d0785e260:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.318+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19.9 2026-03-20T11:46:56.315 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.318+0000 7fa44df93900 20 run: stack=0x558d078599d0 is io blocked 2026-03-20T11:46:56.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.319+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d073d9f80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.319+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d073d9f80:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.319+0000 7fa44df93900 15 stack 0x558d07855790 end 2026-03-20T11:46:56.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.319+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.319+0000 7fa44df93900 20 run: stack=0x558d07855790 is done 2026-03-20T11:46:56.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.319+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c2910:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.319+0000 7fa44df93900 20 run: stack=0x558d0789c890 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.319+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d073dcb00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.319+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d073dcb00:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.319+0000 7fa44df93900 15 stack 0x558d078c7b50 end 2026-03-20T11:46:56.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.319+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.319+0000 7fa44df93900 20 run: stack=0x558d078c7b50 is done 2026-03-20T11:46:56.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.319+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c2910:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.316 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.319+0000 7fa44df93900 20 run: stack=0x558d0789c890 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d0785e260:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d0785e260:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa44df93900 15 stack 0x558d078599d0 end 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa44df93900 20 run: stack=0x558d078599d0 is done 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c2910:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4e40:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa44df93900 20 trim: trimmed bucket instance dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa44df93900 15 stack 0x558d0789c890 end 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa44df93900 20 run: stack=0x558d0789c890 is done 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4d70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa44df93900 4 starting trim on bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa44df93900 20 enqueued request req=0x558d0785d060 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa44df93900 20 req: 0x558d0785d060 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa44df93900 20 run: stack=0x558d0789c890 is io blocked 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa40affd640 20 dequeued request req=0x558d0785d060 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa40affd640 20 RGWWQ: empty 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa40affd640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 : miss 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa40affd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa40affd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa40affd640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 info.flags=0x17 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa40affd640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 to cache LRU end 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa40affd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.317 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa40affd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa40affd640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:46:56.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa40affd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:46:56.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa40affd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-35 : miss 2026-03-20T11:46:56.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa40affd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa40affd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa40affd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-35 info.flags=0x0 2026-03-20T11:46:56.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa40affd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-35 to cache LRU end 2026-03-20T11:46:56.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa40affd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-35 : miss 2026-03-20T11:46:56.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.320+0000 7fa40affd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa40affd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa40affd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-35 info.flags=0x0 2026-03-20T11:46:56.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa40affd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-35 to cache LRU end 2026-03-20T11:46:56.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa40affd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40): adding source pipe: {s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.318 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:56.321+0000 7f126f1ce640 -1 req 6682387124411512051 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:56.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa40affd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40): adding dest pipe: {s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa40affd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa40affd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa40affd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:46:56.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4d70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d076626a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4d70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 20 run: stack=0x558d0789c890 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d078c97b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d078c97b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.318 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = ff5e20387f1424c8b80b24983c9f0f24f4163231920a824bd70baeaabdaedf36 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:ff5e20387f1424c8b80b24983c9f0f24f4163231920a824bd70baeaabdaedf36 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 1b4de7d14af9251f550337cf5255a61d20a3534a84abdc5ccd02973543595e58 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=1b4de7d14af9251f550337cf5255a61d20a3534a84abdc5ccd02973543595e58 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=28, curl_handle=0x558d0786c860 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa44df93900 20 run: stack=0x558d078599d0 is io blocked 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.321+0000 7fa3f0ff9640 20 link_request req_data=0x558d073d4c70 req_data->id=28, curl_handle=0x558d0786c860 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.323+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078a1d80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.319 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.323+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078a1d80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.323+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078a1d80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.323+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078a1d80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.323+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078a4c10:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.323+0000 7fa44df93900 15 stack 0x558d07876860 end 2026-03-20T11:46:56.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.323+0000 7fa44df93900 20 run: stack=0x558d07876860 is done 2026-03-20T11:46:56.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.323+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.323+0000 7fa44df93900 20 collect(): s=0x558d0785c1a0 stack=0x558d07876860 is complete 2026-03-20T11:46:56.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.323+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.323+0000 7fa44df93900 10 trimming bilogs for bucket=:dipepu-36[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41]) markers=[,,,,,,,,,,00000000001.53.6], shards=11 2026-03-20T11:46:56.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.323+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a1d80:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.323+0000 7fa44df93900 10 trimming bilog shard 10 of :dipepu-36[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41]) at marker 00000000001.53.6 2026-03-20T11:46:56.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.323+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.323+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d79b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.323+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d79b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.323+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41.10 2026-03-20T11:46:56.320 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.323+0000 7fa44df93900 20 run: stack=0x558d07876860 is io blocked 2026-03-20T11:46:56.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.324+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d79b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.324+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d79b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.324+0000 7fa44df93900 15 stack 0x558d07876860 end 2026-03-20T11:46:56.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.324+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.324+0000 7fa44df93900 20 run: stack=0x558d07876860 is done 2026-03-20T11:46:56.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.324+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a1d80:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.324+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.324+0000 7fa44df93900 20 trim: trimmed bucket instance dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:46:56.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.324+0000 7fa44df93900 15 stack 0x558d0785c1a0 end 2026-03-20T11:46:56.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.324+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is done 2026-03-20T11:46:56.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.324+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.324+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.324+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.324+0000 7fa44df93900 4 starting trim on bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:46:56.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.324+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078541f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.324+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078541f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.324+0000 7fa44df93900 20 enqueued request req=0x558d073db6d0 2026-03-20T11:46:56.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.324+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.324+0000 7fa44df93900 20 req: 0x558d073db6d0 2026-03-20T11:46:56.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.324+0000 7fa4117fa640 20 dequeued request req=0x558d073db6d0 2026-03-20T11:46:56.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.324+0000 7fa4117fa640 20 RGWWQ: empty 2026-03-20T11:46:56.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.324+0000 7fa4117fa640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 : miss 2026-03-20T11:46:56.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.324+0000 7fa4117fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.321 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.324+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is io blocked 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa4117fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa4117fa640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 info.flags=0x17 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa4117fa640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 to cache LRU end 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa4117fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa4117fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa4117fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa4117fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-26 : miss 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa4117fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078a07e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 15 stack 0x558d07372f40 end 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 run: stack=0x558d07372f40 is done 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 collect(): s=0x558d07856e80 stack=0x558d07372f40 is complete 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 10 trimming bilogs for bucket=:dipepu-22[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27]) markers=[,,,,,,,,00000000005.29.12,,], shards=11 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a1d80:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 10 trimming bilog shard 8 of :dipepu-22[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27]) at marker 00000000005.29.12 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 run: stack=0x558d07856e80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078ad160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078ad160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27.8 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 run: stack=0x558d07372f40 is io blocked 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa4117fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa4117fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-26 info.flags=0x0 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa4117fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-26 to cache LRU end 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa4117fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-26 : miss 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa4117fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa4117fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa4117fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-26 info.flags=0x0 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa4117fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-26 to cache LRU end 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa4117fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31): adding source pipe: {s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa4117fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31): adding dest pipe: {s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa4117fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa4117fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa4117fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078541f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078541f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078541f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.322 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078541f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078a07e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = eced813417f90d6aebb1e6aa830259c03f97fcbe00bce7a270c5014fcd9681b2 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:eced813417f90d6aebb1e6aa830259c03f97fcbe00bce7a270c5014fcd9681b2 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 6ff10d0cca91a5c11bdfe86850d009f6e90956ebf277c8590c84c7ff18794205 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=6ff10d0cca91a5c11bdfe86850d009f6e90956ebf277c8590c84c7ff18794205 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=29, curl_handle=0x558d0785fe10 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa44df93900 20 run: stack=0x558d07876860 is io blocked 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.325+0000 7fa3f0ff9640 20 link_request req_data=0x558d073daf10 req_data->id=29, curl_handle=0x558d0785fe10 2026-03-20T11:46:56.323 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:56.326+0000 7f12671be640 -1 req 2091577348734029625 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.326+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078ad160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.326+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078ad160:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.326+0000 7fa44df93900 15 stack 0x558d07372f40 end 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.326+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.323 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.326+0000 7fa44df93900 20 run: stack=0x558d07372f40 is done 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.326+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a1d80:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.326+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.326+0000 7fa44df93900 20 trim: trimmed bucket instance dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.326+0000 7fa44df93900 15 stack 0x558d07856e80 end 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.326+0000 7fa44df93900 20 run: stack=0x558d07856e80 is done 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.326+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.326+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.326+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.326+0000 7fa44df93900 4 starting trim on bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.326+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078ad160:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.326+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078ad160:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.326+0000 7fa44df93900 20 enqueued request req=0x558d078abc30 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.326+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.326+0000 7fa44df93900 20 req: 0x558d078abc30 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.326+0000 7fa410ff9640 20 dequeued request req=0x558d078abc30 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.326+0000 7fa410ff9640 20 RGWWQ: empty 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.326+0000 7fa410ff9640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 : miss 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.326+0000 7fa410ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.326+0000 7fa44df93900 20 run: stack=0x558d07856e80 is io blocked 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa410ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa410ff9640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 info.flags=0x17 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa410ff9640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 to cache LRU end 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa410ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa410ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa410ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa410ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-51 : miss 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa410ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa410ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa410ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-51 info.flags=0x0 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa410ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-51 to cache LRU end 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa410ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-51 : miss 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa410ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa410ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa410ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-51 info.flags=0x0 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa410ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-51 to cache LRU end 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa410ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11): adding source pipe: {s={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa410ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11): adding dest pipe: {s={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa410ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa410ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa410ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078ad160:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078ad160:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078ad160:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078ad160:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078a1d80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.324 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 20 run: stack=0x558d07856e80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d07866420:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d07866420:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 5e48fdbfe0ff85c1d1cb246fe5bf24c272f2051cd2afc063bccda4eee63b9090 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:5e48fdbfe0ff85c1d1cb246fe5bf24c272f2051cd2afc063bccda4eee63b9090 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 506bf6f01ef527a0a16091ec24b3c781104ca6aa641111c65131afb9246232ab 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=506bf6f01ef527a0a16091ec24b3c781104ca6aa641111c65131afb9246232ab 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=30, curl_handle=0x558d07868640 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa44df93900 20 run: stack=0x558d07372f40 is io blocked 2026-03-20T11:46:56.325 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.327+0000 7fa3f0ff9640 20 link_request req_data=0x558d07856be0 req_data->id=30, curl_handle=0x558d07868640 2026-03-20T11:46:56.332 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07855030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.332 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07855030:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:56.332 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d073d8940:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa44df93900 15 stack 0x558d078a1890 end 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa44df93900 20 run: stack=0x558d078a1890 is done 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa44df93900 20 collect(): s=0x558d0786bfc0 stack=0x558d078a1890 is complete 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa44df93900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa44df93900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa44df93900 15 stack 0x558d0786bfc0 end 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa44df93900 20 stack->operate() returned ret=-22 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is done 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa44df93900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa44df93900 4 starting trim on bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07666b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07666b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa44df93900 20 enqueued request req=0x558d07863d20 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa44df93900 20 req: 0x558d07863d20 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa40bfff640 20 dequeued request req=0x558d07863d20 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa40bfff640 20 RGWWQ: empty 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa40bfff640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 : miss 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa40bfff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is io blocked 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa40bfff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa40bfff640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 info.flags=0x17 2026-03-20T11:46:56.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa40bfff640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 to cache LRU end 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa40bfff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa40bfff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa40bfff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa40bfff640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-55 : miss 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa40bfff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa40bfff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa40bfff640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-55 info.flags=0x0 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa40bfff640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-55 to cache LRU end 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa40bfff640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-55 : miss 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa40bfff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa40bfff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa40bfff640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-55 info.flags=0x0 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.336+0000 7fa40bfff640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-55 to cache LRU end 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa40bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding source pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa40bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding dest pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa40bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa40bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa40bfff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07666b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07666b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07666b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07666b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d073d8940:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07855030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07855030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = cfab0588c824c1030799245293c5490b380c06ff3bb488221919e84dedde8753 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:cfab0588c824c1030799245293c5490b380c06ff3bb488221919e84dedde8753 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.335 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = d53b2faf66731ca5591696601c30017eb3a7b9265d5fb3ce81c6f8976691be7f 2026-03-20T11:46:56.335 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=d53b2faf66731ca5591696601c30017eb3a7b9265d5fb3ce81c6f8976691be7f 2026-03-20T11:46:56.335 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.335 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.335 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.335 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=31, curl_handle=0x558d07864550 2026-03-20T11:46:56.335 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa44df93900 20 run: stack=0x558d078a1890 is io blocked 2026-03-20T11:46:56.335 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.337+0000 7fa3f0ff9640 20 link_request req_data=0x558d078a55e0 req_data->id=31, curl_handle=0x558d07864550 2026-03-20T11:46:56.358 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d078c97b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d078c97b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d076626a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa44df93900 15 stack 0x558d078599d0 end 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa44df93900 20 run: stack=0x558d078599d0 is done 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4d70:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa44df93900 20 collect(): s=0x558d0789c890 stack=0x558d078599d0 is complete 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa44df93900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa44df93900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4d70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa44df93900 15 stack 0x558d0789c890 end 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa44df93900 20 stack->operate() returned ret=-22 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa44df93900 20 run: stack=0x558d0789c890 is done 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa44df93900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa44df93900 4 starting trim on bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078af210:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078af210:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa44df93900 20 enqueued request req=0x558d0785d060 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa44df93900 20 req: 0x558d0785d060 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa40b7fe640 20 dequeued request req=0x558d0785d060 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa40b7fe640 20 RGWWQ: empty 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa40b7fe640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 : miss 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa40b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa44df93900 20 run: stack=0x558d0789c890 is io blocked 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa40b7fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa40b7fe640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 info.flags=0x17 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa40b7fe640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 to cache LRU end 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa40b7fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa40b7fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa40b7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa40b7fe640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-28 : miss 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa40b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa40b7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa40b7fe640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-28 info.flags=0x0 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa40b7fe640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-28 to cache LRU end 2026-03-20T11:46:56.359 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa40b7fe640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-28 : miss 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.362+0000 7fa40b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa40b7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa40b7fe640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-28 info.flags=0x0 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa40b7fe640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-28 to cache LRU end 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa40b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding source pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa40b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding dest pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa40b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa40b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa40b7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078af210:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078af210:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078af210:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078af210:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d076626a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 20 run: stack=0x558d0789c890 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d07663f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d07663f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 65463f6c37624f12d13ff6b01a15c9c9413242c8d8e592f4802737fb2a0693e5 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:65463f6c37624f12d13ff6b01a15c9c9413242c8d8e592f4802737fb2a0693e5 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.360 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = a0145981a5bd1efe7547956fe1139a2432bbdf04d4e0b0e72f5c20f7c15a281b 2026-03-20T11:46:56.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a0145981a5bd1efe7547956fe1139a2432bbdf04d4e0b0e72f5c20f7c15a281b 2026-03-20T11:46:56.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=32, curl_handle=0x558d0786c860 2026-03-20T11:46:56.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa44df93900 20 run: stack=0x558d078599d0 is io blocked 2026-03-20T11:46:56.361 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.363+0000 7fa3f0ff9640 20 link_request req_data=0x558d07868b70 req_data->id=32, curl_handle=0x558d0786c860 2026-03-20T11:46:56.361 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:56.363+0000 7f1231953640 -1 req 4250034523124968314 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:56.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.363 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078a07e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa44df93900 15 stack 0x558d07876860 end 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa44df93900 20 run: stack=0x558d07876860 is done 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa44df93900 20 collect(): s=0x558d0785c1a0 stack=0x558d07876860 is complete 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa44df93900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa44df93900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa44df93900 15 stack 0x558d0785c1a0 end 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa44df93900 20 stack->operate() returned ret=-22 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is done 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa44df93900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa44df93900 4 starting trim on bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078af210:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078af210:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa44df93900 20 enqueued request req=0x558d073db6d0 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa44df93900 20 req: 0x558d073db6d0 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa3f8fe9640 20 dequeued request req=0x558d073db6d0 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa3f8fe9640 20 RGWWQ: empty 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa3f8fe9640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 : miss 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa3f8fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is io blocked 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa3f8fe9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa3f8fe9640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 info.flags=0x17 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa3f8fe9640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 to cache LRU end 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa3f8fe9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa3f8fe9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa3f8fe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa3f8fe9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-31 : miss 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.367+0000 7fa3f8fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa3f8fe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa3f8fe9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-31 info.flags=0x0 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa3f8fe9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-31 to cache LRU end 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa3f8fe9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-31 : miss 2026-03-20T11:46:56.364 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa3f8fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa3f8fe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa3f8fe9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-31 info.flags=0x0 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa3f8fe9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-31 to cache LRU end 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa3f8fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding source pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa3f8fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding dest pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa3f8fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa3f8fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa3f8fe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078af210:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078af210:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078af210:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078af210:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078a07e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.365 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.366 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 6705cb7fab75ef2724ba38a0bcf01639e5994d442da67fa22c9620bfb2e869a6 2026-03-20T11:46:56.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.366 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.366 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.366 INFO:teuthology.orchestra.run.vm02.stderr:6705cb7fab75ef2724ba38a0bcf01639e5994d442da67fa22c9620bfb2e869a6 2026-03-20T11:46:56.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 4d58f64e00ea80306980a10223269ed9f6f64fdc2466c686f6d39d35932f8054 2026-03-20T11:46:56.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=4d58f64e00ea80306980a10223269ed9f6f64fdc2466c686f6d39d35932f8054 2026-03-20T11:46:56.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=33, curl_handle=0x558d0785fe10 2026-03-20T11:46:56.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa44df93900 20 run: stack=0x558d07876860 is io blocked 2026-03-20T11:46:56.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.368+0000 7fa3f0ff9640 20 link_request req_data=0x558d073daf10 req_data->id=33, curl_handle=0x558d0785fe10 2026-03-20T11:46:56.366 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:56.369+0000 7f122a945640 -1 req 5311769147497918220 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:56.366 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d07866420:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d07866420:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d07866420:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d07866420:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078a1d80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 15 stack 0x558d07372f40 end 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 20 run: stack=0x558d07372f40 is done 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 20 collect(): s=0x558d07856e80 stack=0x558d07372f40 is complete 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 10 trimming bilogs for bucket=:dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) markers=[00000000002.62.6,,00000000002.48.6,,00000000002.70.6,,,00000000002.59.6,,,], shards=11 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a1d80:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 10 trimming bilog shard 0 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.62.6 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 10 trimming bilog shard 2 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.48.6 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 10 trimming bilog shard 4 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.70.6 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 10 trimming bilog shard 7 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.59.6 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 20 run: stack=0x558d07856e80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078ad160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d073dbd40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d07664e50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d073dc7e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078ad160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.0 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 20 run: stack=0x558d07372f40 is io blocked 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d073dbd40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.2 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 20 run: stack=0x558d078c7b50 is io blocked 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d07664e50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.4 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 20 run: stack=0x558d07855790 is io blocked 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d073dc7e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.7 2026-03-20T11:46:56.367 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.370+0000 7fa44df93900 20 run: stack=0x558d0786aaf0 is io blocked 2026-03-20T11:46:56.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.371+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d07664e50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.371+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d07664e50:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.371+0000 7fa44df93900 15 stack 0x558d07855790 end 2026-03-20T11:46:56.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.371+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.371+0000 7fa44df93900 20 run: stack=0x558d07855790 is done 2026-03-20T11:46:56.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.371+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a1d80:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.371+0000 7fa44df93900 20 run: stack=0x558d07856e80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.371+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078ad160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.371+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078ad160:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.371+0000 7fa44df93900 15 stack 0x558d07372f40 end 2026-03-20T11:46:56.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.371+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.371+0000 7fa44df93900 20 run: stack=0x558d07372f40 is done 2026-03-20T11:46:56.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.371+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a1d80:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.371+0000 7fa44df93900 20 run: stack=0x558d07856e80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.371+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d073dc7e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.371+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d073dc7e0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.371+0000 7fa44df93900 15 stack 0x558d0786aaf0 end 2026-03-20T11:46:56.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.371+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.371+0000 7fa44df93900 20 run: stack=0x558d0786aaf0 is done 2026-03-20T11:46:56.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.371+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a1d80:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.368 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.371+0000 7fa44df93900 20 run: stack=0x558d07856e80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d073dbd40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d073dbd40:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa44df93900 15 stack 0x558d078c7b50 end 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa44df93900 20 run: stack=0x558d078c7b50 is done 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a1d80:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa44df93900 20 trim: trimmed bucket instance dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa44df93900 15 stack 0x558d07856e80 end 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa44df93900 20 run: stack=0x558d07856e80 is done 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa44df93900 4 starting trim on bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078c4330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078c4330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa44df93900 20 enqueued request req=0x558d07876670 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa44df93900 20 req: 0x558d07876670 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa44df93900 20 run: stack=0x558d07856e80 is io blocked 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa40a7fc640 20 dequeued request req=0x558d07876670 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa40a7fc640 20 RGWWQ: empty 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa40a7fc640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 : miss 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa40a7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa40a7fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa40a7fc640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 info.flags=0x17 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa40a7fc640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 to cache LRU end 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa40a7fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa40a7fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa40a7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa40a7fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-39 : miss 2026-03-20T11:46:56.370 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.373+0000 7fa40a7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.374+0000 7fa40a7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.374+0000 7fa40a7fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-39 info.flags=0x0 2026-03-20T11:46:56.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.374+0000 7fa40a7fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-39 to cache LRU end 2026-03-20T11:46:56.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.374+0000 7fa40a7fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-39 : miss 2026-03-20T11:46:56.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.374+0000 7fa40a7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.374+0000 7fa40a7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.374+0000 7fa40a7fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-39 info.flags=0x0 2026-03-20T11:46:56.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.374+0000 7fa40a7fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-39 to cache LRU end 2026-03-20T11:46:56.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.374+0000 7fa40a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding source pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.374+0000 7fa40a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding dest pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.374+0000 7fa40a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.374+0000 7fa40a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.374+0000 7fa40a7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:56.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.374+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078c4330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.371 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.374+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078c4330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.374+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078c4330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.374+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078c4330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.374+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.374+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d078a1d80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.374+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.374+0000 7fa44df93900 20 run: stack=0x558d07856e80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.374+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07866420:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.374+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07866420:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.374+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.374+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.374+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.375+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.375+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.375+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.375+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 27cce7773f98aaa53ed460b8cf1cc13990777a148ae5d6cc2adbbdffa76ad76b 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.375+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:27cce7773f98aaa53ed460b8cf1cc13990777a148ae5d6cc2adbbdffa76ad76b 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.375+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.375+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.375+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.375+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.375+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = bf996e32b57b30b44f66f26d02cc9d7b06fb0be8a03e3e246f6fec6a5e979159 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.375+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=bf996e32b57b30b44f66f26d02cc9d7b06fb0be8a03e3e246f6fec6a5e979159 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.375+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.375+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.375+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.375+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=34, curl_handle=0x558d07868640 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.375+0000 7fa44df93900 20 run: stack=0x558d078c7b50 is io blocked 2026-03-20T11:46:56.372 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.375+0000 7fa3f0ff9640 20 link_request req_data=0x558d073d4c70 req_data->id=34, curl_handle=0x558d07868640 2026-03-20T11:46:56.375 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.379+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07855030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.379+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07855030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.379+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07855030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.379+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07855030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.379+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d073d8940:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.379+0000 7fa44df93900 15 stack 0x558d078a1890 end 2026-03-20T11:46:56.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.379+0000 7fa44df93900 20 run: stack=0x558d078a1890 is done 2026-03-20T11:46:56.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.379+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.379+0000 7fa44df93900 20 collect(): s=0x558d0786bfc0 stack=0x558d078a1890 is complete 2026-03-20T11:46:56.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.379+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.379+0000 7fa44df93900 10 trimming bilogs for bucket=:dipepu-55[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15]) markers=[,,,,,00000000001.133.6,,,,,], shards=11 2026-03-20T11:46:56.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.379+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07855030:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.379+0000 7fa44df93900 10 trimming bilog shard 5 of :dipepu-55[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15]) at marker 00000000001.133.6 2026-03-20T11:46:56.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.379+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.379+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07666b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.379+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07666b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.379+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15.5 2026-03-20T11:46:56.376 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.379+0000 7fa44df93900 20 run: stack=0x558d078a1890 is io blocked 2026-03-20T11:46:56.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07666b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07666b50:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa44df93900 15 stack 0x558d078a1890 end 2026-03-20T11:46:56.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa44df93900 20 run: stack=0x558d078a1890 is done 2026-03-20T11:46:56.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07855030:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa44df93900 20 trim: trimmed bucket instance dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:46:56.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa44df93900 15 stack 0x558d0786bfc0 end 2026-03-20T11:46:56.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is done 2026-03-20T11:46:56.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa44df93900 4 starting trim on bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:56.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d0786fee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d0786fee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa44df93900 20 enqueued request req=0x558d07863d20 2026-03-20T11:46:56.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa44df93900 20 req: 0x558d07863d20 2026-03-20T11:46:56.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is io blocked 2026-03-20T11:46:56.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa409ffb640 20 dequeued request req=0x558d07863d20 2026-03-20T11:46:56.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa409ffb640 20 RGWWQ: empty 2026-03-20T11:46:56.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa409ffb640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 : miss 2026-03-20T11:46:56.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa409ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa409ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:46:56.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa409ffb640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 info.flags=0x17 2026-03-20T11:46:56.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa409ffb640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 to cache LRU end 2026-03-20T11:46:56.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa409ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa409ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa409ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:56.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa409ffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-8 : miss 2026-03-20T11:46:56.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.381+0000 7fa409ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa409ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa409ffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-8 info.flags=0x0 2026-03-20T11:46:56.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa409ffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-8 to cache LRU end 2026-03-20T11:46:56.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa409ffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-8 : miss 2026-03-20T11:46:56.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa409ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa409ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa409ffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-8 info.flags=0x0 2026-03-20T11:46:56.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa409ffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-8 to cache LRU end 2026-03-20T11:46:56.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa409ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding source pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa409ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding dest pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa409ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa409ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa409ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:56.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d0786fee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d0786fee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d0786fee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d0786fee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d078c0fc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07855030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07855030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 144287c94b8f01d529c213ae1bf7288fc381c727a918dc54cc65e960e3097878 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:144287c94b8f01d529c213ae1bf7288fc381c727a918dc54cc65e960e3097878 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 071bb242986a8ff6fc74302fbbe1fa5ed34be1312425e088dae517e2cc5bc4e3 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=071bb242986a8ff6fc74302fbbe1fa5ed34be1312425e088dae517e2cc5bc4e3 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=35, curl_handle=0x558d07864550 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa44df93900 20 run: stack=0x558d078a1890 is io blocked 2026-03-20T11:46:56.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.382+0000 7fa3f0ff9640 20 link_request req_data=0x558d078a55e0 req_data->id=35, curl_handle=0x558d07864550 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d07663f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d07663f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d076626a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa44df93900 15 stack 0x558d078599d0 end 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa44df93900 20 run: stack=0x558d078599d0 is done 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa44df93900 20 collect(): s=0x558d0789c890 stack=0x558d078599d0 is complete 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa44df93900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa44df93900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4aa0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa44df93900 15 stack 0x558d0789c890 end 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa44df93900 20 stack->operate() returned ret=-22 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa44df93900 20 run: stack=0x558d0789c890 is done 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa44df93900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa44df93900 4 starting trim on bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d0786fee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d0786fee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa44df93900 20 enqueued request req=0x558d0785d060 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa44df93900 20 req: 0x558d0785d060 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa44df93900 20 run: stack=0x558d0789c890 is io blocked 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa4097fa640 20 dequeued request req=0x558d0785d060 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa4097fa640 20 RGWWQ: empty 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa4097fa640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 : miss 2026-03-20T11:46:56.401 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa4097fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa4097fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa4097fa640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 info.flags=0x17 2026-03-20T11:46:56.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa4097fa640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 to cache LRU end 2026-03-20T11:46:56.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa4097fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa4097fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa4097fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:56.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa4097fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-16 : miss 2026-03-20T11:46:56.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa4097fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa4097fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa4097fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-16 info.flags=0x0 2026-03-20T11:46:56.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa4097fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-16 to cache LRU end 2026-03-20T11:46:56.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa4097fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-16 : miss 2026-03-20T11:46:56.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.404+0000 7fa4097fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa4097fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa4097fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-16 info.flags=0x0 2026-03-20T11:46:56.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa4097fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-16 to cache LRU end 2026-03-20T11:46:56.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa4097fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding source pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa4097fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding dest pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa4097fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.402 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa4097fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa4097fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d0786fee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d0786fee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d0786fee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d0786fee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d076626a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 20 run: stack=0x558d0789c890 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d07663f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d07663f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = b4d808aa76ee2584d3c03508e56dad5f0f9170f48a266eabd92022a439444621 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:b4d808aa76ee2584d3c03508e56dad5f0f9170f48a266eabd92022a439444621 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = b3e7c236c3443c7f45a6a221586d4df98a2f84b16265ddc84e61e80925579562 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b3e7c236c3443c7f45a6a221586d4df98a2f84b16265ddc84e61e80925579562 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=36, curl_handle=0x558d0786c860 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa44df93900 20 run: stack=0x558d078599d0 is io blocked 2026-03-20T11:46:56.403 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.405+0000 7fa3f0ff9640 20 link_request req_data=0x558d07868b70 req_data->id=36, curl_handle=0x558d0786c860 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078a07e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa44df93900 15 stack 0x558d07876860 end 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa44df93900 20 run: stack=0x558d07876860 is done 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa44df93900 20 collect(): s=0x558d0785c1a0 stack=0x558d07876860 is complete 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa44df93900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa44df93900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa44df93900 15 stack 0x558d0785c1a0 end 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa44df93900 20 stack->operate() returned ret=-22 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is done 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa44df93900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa44df93900 4 starting trim on bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d0786fee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d0786fee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa44df93900 20 enqueued request req=0x558d073db6d0 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa44df93900 20 req: 0x558d073db6d0 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is io blocked 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa408ff9640 20 dequeued request req=0x558d073db6d0 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa408ff9640 20 RGWWQ: empty 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa408ff9640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 : miss 2026-03-20T11:46:56.407 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa408ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa408ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa408ff9640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 info.flags=0x17 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa408ff9640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 to cache LRU end 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa408ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa408ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa408ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa408ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-43 : miss 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa408ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa408ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa408ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-43 info.flags=0x0 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa408ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-43 to cache LRU end 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa408ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-43 : miss 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa408ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa408ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa408ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-43 info.flags=0x0 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa408ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-43 to cache LRU end 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa408ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding source pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.410+0000 7fa408ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding dest pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa408ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa408ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa408ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d0786fee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d0786fee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d0786fee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d0786fee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078a07e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.408 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.409 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.409 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.409 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 290987aba5994259ac98c5bd3653fc9e48b065060711d7b0113d6bc5ba9633ed 2026-03-20T11:46:56.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.409 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.409 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.409 INFO:teuthology.orchestra.run.vm02.stderr:290987aba5994259ac98c5bd3653fc9e48b065060711d7b0113d6bc5ba9633ed 2026-03-20T11:46:56.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 74cc2faaa486ffd2df352cdeb6f2dde1f089f92f89232f9fd3ba97aa30f2548e 2026-03-20T11:46:56.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=74cc2faaa486ffd2df352cdeb6f2dde1f089f92f89232f9fd3ba97aa30f2548e 2026-03-20T11:46:56.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=37, curl_handle=0x558d0785fe10 2026-03-20T11:46:56.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa44df93900 20 run: stack=0x558d07876860 is io blocked 2026-03-20T11:46:56.409 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.411+0000 7fa3f0ff9640 20 link_request req_data=0x558d073daf10 req_data->id=37, curl_handle=0x558d0785fe10 2026-03-20T11:46:56.409 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:56.411+0000 7f1234158640 -1 req 16005699919609334325 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:56.413 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.417+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07866420:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.414 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.417+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07866420:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.414 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.417+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07866420:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.414 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.417+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07866420:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.414 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.417+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d078a1d80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.414 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.417+0000 7fa44df93900 15 stack 0x558d078c7b50 end 2026-03-20T11:46:56.414 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.417+0000 7fa44df93900 20 run: stack=0x558d078c7b50 is done 2026-03-20T11:46:56.414 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.417+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.414 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.417+0000 7fa44df93900 20 collect(): s=0x558d07856e80 stack=0x558d078c7b50 is complete 2026-03-20T11:46:56.414 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.417+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.414 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.417+0000 7fa44df93900 10 trimming bilogs for bucket=:dipepu-39[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44]) markers=[,,,,,,,,00000000001.18.6,,], shards=11 2026-03-20T11:46:56.414 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.417+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a1d80:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.414 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.417+0000 7fa44df93900 10 trimming bilog shard 8 of :dipepu-39[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44]) at marker 00000000001.18.6 2026-03-20T11:46:56.414 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.417+0000 7fa44df93900 20 run: stack=0x558d07856e80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.414 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.417+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d078ad160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.414 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.417+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d078ad160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.414 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.417+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44.8 2026-03-20T11:46:56.414 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.417+0000 7fa44df93900 20 run: stack=0x558d078c7b50 is io blocked 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d078ad160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d078ad160:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa44df93900 15 stack 0x558d078c7b50 end 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa44df93900 20 run: stack=0x558d078c7b50 is done 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a1d80:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa44df93900 20 trim: trimmed bucket instance dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa44df93900 15 stack 0x558d07856e80 end 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa44df93900 20 run: stack=0x558d07856e80 is done 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa44df93900 4 starting trim on bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078ad160:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078ad160:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa44df93900 20 enqueued request req=0x558d07876670 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa44df93900 20 req: 0x558d07876670 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa44df93900 20 run: stack=0x558d07856e80 is io blocked 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa403fff640 20 dequeued request req=0x558d07876670 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa403fff640 20 RGWWQ: empty 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa403fff640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 : miss 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa403fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa403fff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa403fff640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 info.flags=0x17 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa403fff640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 to cache LRU end 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa403fff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa403fff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa403fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa403fff640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-10 : miss 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.418+0000 7fa403fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.415 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.419+0000 7fa403fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.416 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.419+0000 7fa403fff640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-10 info.flags=0x0 2026-03-20T11:46:56.416 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.419+0000 7fa403fff640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-10 to cache LRU end 2026-03-20T11:46:56.416 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.419+0000 7fa403fff640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-10 : miss 2026-03-20T11:46:56.416 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.419+0000 7fa403fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.416 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.419+0000 7fa403fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.416 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.419+0000 7fa403fff640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-10 info.flags=0x0 2026-03-20T11:46:56.416 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.419+0000 7fa403fff640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-10 to cache LRU end 2026-03-20T11:46:56.416 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.419+0000 7fa403fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding source pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.416 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.419+0000 7fa403fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding dest pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.416 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.419+0000 7fa403fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.416 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.419+0000 7fa403fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.416 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.419+0000 7fa403fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.419+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078ad160:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.419+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078ad160:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.419+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078ad160:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.419+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078ad160:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.419+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.419+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07854960:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.419+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.419+0000 7fa44df93900 20 run: stack=0x558d07856e80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.419+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07866420:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.419+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07866420:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.419+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.419+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.419+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.420+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.420+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.420+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.420+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 742bae26d34aa8b35b663c42a9720b614dc374905ec03d654c602e39385f5baf 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.420+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:742bae26d34aa8b35b663c42a9720b614dc374905ec03d654c602e39385f5baf 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.420+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.420+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.420+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.420+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.420+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 6fe308f8ccf62a6d0d54c5faf27cd5bc465bd852dee800b6bc2b1c322f496602 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.420+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=6fe308f8ccf62a6d0d54c5faf27cd5bc465bd852dee800b6bc2b1c322f496602 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.420+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.420+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.420+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.420+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=38, curl_handle=0x558d07868640 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.420+0000 7fa44df93900 20 run: stack=0x558d078c7b50 is io blocked 2026-03-20T11:46:56.417 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.420+0000 7fa3f0ff9640 20 link_request req_data=0x558d073d4c70 req_data->id=38, curl_handle=0x558d07868640 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07855030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07855030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07855030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d07855030:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d078c0fc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 15 stack 0x558d078a1890 end 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 run: stack=0x558d078a1890 is done 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 collect(): s=0x558d0786bfc0 stack=0x558d078a1890 is complete 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 10 trimming bilogs for bucket=:dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) markers=[,00000000002.13.6,,00000000001.5.6,,00000000001.2.6,00000000002.13.6,00000000001.11.6,00000000001.4.6,00000000001.12.6,], shards=11 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d073dba40:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 10 trimming bilog shard 1 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000002.13.6 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 10 trimming bilog shard 3 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.5.6 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 10 trimming bilog shard 5 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.2.6 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 10 trimming bilog shard 6 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000002.13.6 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 10 trimming bilog shard 7 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.11.6 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 10 trimming bilog shard 8 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.4.6 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 10 trimming bilog shard 9 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.12.6 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d073d79b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d07666b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078c6650:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d073dc7e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d078c97b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d076674c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 cr:s=0x558d0786fee0:op=0x558d078cbdf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d073d79b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.1 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 run: stack=0x558d078a1890 is io blocked 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d07666b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.3 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 run: stack=0x558d0786aaf0 is io blocked 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078c6650:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.5 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 run: stack=0x558d07372f40 is io blocked 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d073dc7e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.6 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 run: stack=0x558d07855790 is io blocked 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d078c97b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.7 2026-03-20T11:46:56.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 run: stack=0x558d0785c780 is io blocked 2026-03-20T11:46:56.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d076674c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.8 2026-03-20T11:46:56.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 run: stack=0x558d07860730 is io blocked 2026-03-20T11:46:56.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 cr:s=0x558d0786fee0:op=0x558d078cbdf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.9 2026-03-20T11:46:56.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.425+0000 7fa44df93900 20 run: stack=0x558d0786fee0 is io blocked 2026-03-20T11:46:56.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.426+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d07666b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.426+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d07666b50:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.426+0000 7fa44df93900 15 stack 0x558d0786aaf0 end 2026-03-20T11:46:56.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.426+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.426+0000 7fa44df93900 20 run: stack=0x558d0786aaf0 is done 2026-03-20T11:46:56.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.426+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d073dba40:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.426+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.426+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078c6650:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.426+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078c6650:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.426+0000 7fa44df93900 15 stack 0x558d07372f40 end 2026-03-20T11:46:56.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.426+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.426+0000 7fa44df93900 20 run: stack=0x558d07372f40 is done 2026-03-20T11:46:56.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.426+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d073dba40:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.426+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d076674c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d076674c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 15 stack 0x558d07860730 end 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 run: stack=0x558d07860730 is done 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d073dba40:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d073d79b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 cr:s=0x558d078a1890:op=0x558d073d79b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 15 stack 0x558d078a1890 end 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 run: stack=0x558d078a1890 is done 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d073dba40:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d078c97b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 cr:s=0x558d0785c780:op=0x558d078c97b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 15 stack 0x558d0785c780 end 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 run: stack=0x558d0785c780 is done 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d073dba40:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 cr:s=0x558d0786fee0:op=0x558d078cbdf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 cr:s=0x558d0786fee0:op=0x558d078cbdf0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 15 stack 0x558d0786fee0 end 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 run: stack=0x558d0786fee0 is done 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d073dba40:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d073dc7e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 cr:s=0x558d07855790:op=0x558d073dc7e0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 15 stack 0x558d07855790 end 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 run: stack=0x558d07855790 is done 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d073dba40:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 cr:s=0x558d0786bfc0:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 trim: trimmed bucket instance dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 15 stack 0x558d0786bfc0 end 2026-03-20T11:46:56.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 run: stack=0x558d0786bfc0 is done 2026-03-20T11:46:56.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 4 starting trim on bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:56.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d078c0e70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d078c0e70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.427+0000 7fa44df93900 20 enqueued request req=0x558d07859470 2026-03-20T11:46:56.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.428+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.428+0000 7fa44df93900 20 req: 0x558d07859470 2026-03-20T11:46:56.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.428+0000 7fa44df93900 20 run: stack=0x558d07860730 is io blocked 2026-03-20T11:46:56.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.428+0000 7fa4037fe640 20 dequeued request req=0x558d07859470 2026-03-20T11:46:56.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.428+0000 7fa4037fe640 20 RGWWQ: empty 2026-03-20T11:46:56.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.428+0000 7fa4037fe640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 : miss 2026-03-20T11:46:56.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.428+0000 7fa4037fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.428+0000 7fa4037fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:46:56.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.428+0000 7fa4037fe640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 info.flags=0x17 2026-03-20T11:46:56.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.428+0000 7fa4037fe640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 to cache LRU end 2026-03-20T11:46:56.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.428+0000 7fa4037fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.428+0000 7fa4037fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.428+0000 7fa4037fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:56.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.428+0000 7fa4037fe640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-3 : miss 2026-03-20T11:46:56.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.428+0000 7fa4037fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.428+0000 7fa4037fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.428+0000 7fa4037fe640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-3 info.flags=0x0 2026-03-20T11:46:56.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.428+0000 7fa4037fe640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-3 to cache LRU end 2026-03-20T11:46:56.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.428+0000 7fa4037fe640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-3 : miss 2026-03-20T11:46:56.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.428+0000 7fa4037fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa4037fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa4037fe640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-3 info.flags=0x0 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa4037fe640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-3 to cache LRU end 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa4037fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding source pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa4037fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding dest pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa4037fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa4037fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa4037fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d078c0e70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d078c0e70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d078c0e70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d078c0e70:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d076652b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 20 run: stack=0x558d07860730 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d07855010:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d07855010:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 413950906c7334e0daa3b1816607b724576a59de944307cdc588530b0bfcf5f3 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:413950906c7334e0daa3b1816607b724576a59de944307cdc588530b0bfcf5f3 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 814f35e3663b5614e8cf20f977670dbc0b8481590b055c8dca2926333c26d562 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=814f35e3663b5614e8cf20f977670dbc0b8481590b055c8dca2926333c26d562 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=39, curl_handle=0x558d07864550 2026-03-20T11:46:56.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa44df93900 20 run: stack=0x558d07372f40 is io blocked 2026-03-20T11:46:56.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.429+0000 7fa3f0ff9640 20 link_request req_data=0x558d078a55e0 req_data->id=39, curl_handle=0x558d07864550 2026-03-20T11:46:56.427 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:56.429+0000 7f1239963640 -1 req 13122258989761292546 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:56.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.448+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d07663f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.448+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d07663f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.448+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d07663f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.448+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d07663f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.448+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d076626a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.448+0000 7fa44df93900 15 stack 0x558d078599d0 end 2026-03-20T11:46:56.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.448+0000 7fa44df93900 20 run: stack=0x558d078599d0 is done 2026-03-20T11:46:56.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.448+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.448+0000 7fa44df93900 20 collect(): s=0x558d0789c890 stack=0x558d078599d0 is complete 2026-03-20T11:46:56.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.448+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.448+0000 7fa44df93900 10 trimming bilogs for bucket=:dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) markers=[,,,,,00000000002.22.11,,,00000000031.119.12,,], shards=11 2026-03-20T11:46:56.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.448+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07665930:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.448+0000 7fa44df93900 10 trimming bilog shard 5 of :dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) at marker 00000000002.22.11 2026-03-20T11:46:56.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.448+0000 7fa44df93900 10 trimming bilog shard 8 of :dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) at marker 00000000031.119.12 2026-03-20T11:46:56.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.448+0000 7fa44df93900 20 run: stack=0x558d0789c890 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.448+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d07666b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.448+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d076674c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.448+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d07666b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.448+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21.5 2026-03-20T11:46:56.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.448+0000 7fa44df93900 20 run: stack=0x558d078599d0 is io blocked 2026-03-20T11:46:56.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.448+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d076674c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.448+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21.8 2026-03-20T11:46:56.445 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.448+0000 7fa44df93900 20 run: stack=0x558d0786aaf0 is io blocked 2026-03-20T11:46:56.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d07666b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d07666b50:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa44df93900 15 stack 0x558d078599d0 end 2026-03-20T11:46:56.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa44df93900 20 run: stack=0x558d078599d0 is done 2026-03-20T11:46:56.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07665930:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa44df93900 20 run: stack=0x558d0789c890 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d076674c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d076674c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa44df93900 15 stack 0x558d0786aaf0 end 2026-03-20T11:46:56.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa44df93900 20 run: stack=0x558d0786aaf0 is done 2026-03-20T11:46:56.446 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d07665930:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa44df93900 20 trim: trimmed bucket instance dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa44df93900 15 stack 0x558d0789c890 end 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa44df93900 20 run: stack=0x558d0789c890 is done 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa44df93900 4 starting trim on bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa44df93900 20 enqueued request req=0x558d07863d20 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa44df93900 20 req: 0x558d07863d20 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa44df93900 20 run: stack=0x558d0789c890 is io blocked 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa402ffd640 20 dequeued request req=0x558d07863d20 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa402ffd640 20 RGWWQ: empty 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa402ffd640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 : miss 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.449+0000 7fa402ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.450+0000 7fa402ffd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.450+0000 7fa402ffd640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 info.flags=0x17 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.450+0000 7fa402ffd640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 to cache LRU end 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.450+0000 7fa402ffd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.450+0000 7fa402ffd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.450+0000 7fa402ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.450+0000 7fa402ffd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-9 : miss 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.450+0000 7fa402ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.450+0000 7fa402ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.450+0000 7fa402ffd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-9 info.flags=0x0 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.450+0000 7fa402ffd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-9 to cache LRU end 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.450+0000 7fa402ffd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-9 : miss 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.450+0000 7fa402ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa402ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa402ffd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-9 info.flags=0x0 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa402ffd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-9 to cache LRU end 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa402ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding source pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.447 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa402ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding dest pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa402ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa402ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa402ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d076626a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 20 run: stack=0x558d0789c890 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078cb1f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078cb1f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.448 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.449 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.449 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.449 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.449 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = e42d9d8af3941ff4fd8332823d3fc12c699b4b268b8dd495ced678d84d1eaeff 2026-03-20T11:46:56.449 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.449 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.449 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.449 INFO:teuthology.orchestra.run.vm02.stderr:e42d9d8af3941ff4fd8332823d3fc12c699b4b268b8dd495ced678d84d1eaeff 2026-03-20T11:46:56.449 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.449 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.449 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.449 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.449 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 228180e0dfd627cb82856f7b321a142f48651b74b34aed0ede1a16f17150aa67 2026-03-20T11:46:56.449 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=228180e0dfd627cb82856f7b321a142f48651b74b34aed0ede1a16f17150aa67 2026-03-20T11:46:56.449 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.449 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.449 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.449 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=40, curl_handle=0x558d0786c860 2026-03-20T11:46:56.449 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa44df93900 20 run: stack=0x558d0786aaf0 is io blocked 2026-03-20T11:46:56.449 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.451+0000 7fa3f0ff9640 20 link_request req_data=0x558d07868b70 req_data->id=40, curl_handle=0x558d0786c860 2026-03-20T11:46:56.449 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.449 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d073d5720:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:56.449 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078a07e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.449 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa44df93900 15 stack 0x558d07876860 end 2026-03-20T11:46:56.449 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa44df93900 20 run: stack=0x558d07876860 is done 2026-03-20T11:46:56.449 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa44df93900 20 collect(): s=0x558d0785c1a0 stack=0x558d07876860 is complete 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa44df93900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa44df93900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa44df93900 15 stack 0x558d0785c1a0 end 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa44df93900 20 stack->operate() returned ret=-22 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is done 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa44df93900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa44df93900 4 starting trim on bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa44df93900 20 enqueued request req=0x558d073db6d0 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa44df93900 20 req: 0x558d073db6d0 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is io blocked 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa4027fc640 20 dequeued request req=0x558d073db6d0 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa4027fc640 20 RGWWQ: empty 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa4027fc640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 : miss 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa4027fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa4027fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa4027fc640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 info.flags=0x17 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa4027fc640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 to cache LRU end 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa4027fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa4027fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa4027fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa4027fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-38 : miss 2026-03-20T11:46:56.450 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.452+0000 7fa4027fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa4027fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa4027fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-38 info.flags=0x0 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa4027fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-38 to cache LRU end 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa4027fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-38 : miss 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa4027fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa4027fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa4027fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-38 info.flags=0x0 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa4027fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-38 to cache LRU end 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa4027fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding source pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa4027fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding dest pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa4027fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa4027fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa4027fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078a07e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078c6e80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078c6e80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.451 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = b8a912cb10a55faa8a5b9ba102d71145cc5c276c27a0c54924afdd9fd960ffe8 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr:b8a912cb10a55faa8a5b9ba102d71145cc5c276c27a0c54924afdd9fd960ffe8 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 7e2c09ebfe4f1b5a4efb8cc9d47b83115f3d7a75b417a6b963f9790583f7c897 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=7e2c09ebfe4f1b5a4efb8cc9d47b83115f3d7a75b417a6b963f9790583f7c897 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=41, curl_handle=0x558d0785fe10 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa44df93900 20 run: stack=0x558d07876860 is io blocked 2026-03-20T11:46:56.452 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.453+0000 7fa3f0ff9640 20 link_request req_data=0x558d073daf10 req_data->id=41, curl_handle=0x558d0785fe10 2026-03-20T11:46:56.460 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.463+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07866420:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.460 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.463+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07866420:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.460 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.463+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07866420:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.460 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.463+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07866420:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.460 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.463+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07854960:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.460 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.463+0000 7fa44df93900 15 stack 0x558d078c7b50 end 2026-03-20T11:46:56.460 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.463+0000 7fa44df93900 20 run: stack=0x558d078c7b50 is done 2026-03-20T11:46:56.460 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.463+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.460 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.463+0000 7fa44df93900 20 collect(): s=0x558d07856e80 stack=0x558d078c7b50 is complete 2026-03-20T11:46:56.460 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.463+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.460 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.463+0000 7fa44df93900 10 trimming bilogs for bucket=:dipepu-10[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15]) markers=[,00000000002.9.6,,,,,,,,,], shards=11 2026-03-20T11:46:56.460 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.463+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d07866420:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.460 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.463+0000 7fa44df93900 10 trimming bilog shard 1 of :dipepu-10[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15]) at marker 00000000002.9.6 2026-03-20T11:46:56.460 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.463+0000 7fa44df93900 20 run: stack=0x558d07856e80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.460 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.463+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07666b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.460 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.463+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07666b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.460 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.463+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15.1 2026-03-20T11:46:56.460 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.463+0000 7fa44df93900 20 run: stack=0x558d078c7b50 is io blocked 2026-03-20T11:46:56.461 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.464+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07666b50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.461 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.464+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07666b50:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.461 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.464+0000 7fa44df93900 15 stack 0x558d078c7b50 end 2026-03-20T11:46:56.461 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.464+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.461 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.464+0000 7fa44df93900 20 run: stack=0x558d078c7b50 is done 2026-03-20T11:46:56.461 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.464+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d07866420:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.461 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.464+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.461 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.464+0000 7fa44df93900 20 trim: trimmed bucket instance dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:46:56.461 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.464+0000 7fa44df93900 15 stack 0x558d07856e80 end 2026-03-20T11:46:56.461 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.464+0000 7fa44df93900 20 run: stack=0x558d07856e80 is done 2026-03-20T11:46:56.461 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.464+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.461 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.464+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.461 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.464+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.461 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.464+0000 7fa44df93900 4 starting trim on bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:56.461 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.464+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.461 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.464+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.461 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.464+0000 7fa44df93900 20 enqueued request req=0x558d07876670 2026-03-20T11:46:56.461 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.464+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.461 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.464+0000 7fa44df93900 20 req: 0x558d07876670 2026-03-20T11:46:56.461 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.464+0000 7fa44df93900 20 run: stack=0x558d07856e80 is io blocked 2026-03-20T11:46:56.461 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.464+0000 7fa401ffb640 20 dequeued request req=0x558d07876670 2026-03-20T11:46:56.461 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.464+0000 7fa401ffb640 20 RGWWQ: empty 2026-03-20T11:46:56.461 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.464+0000 7fa401ffb640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 : miss 2026-03-20T11:46:56.461 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.464+0000 7fa401ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.462 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.465+0000 7fa401ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.462 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.465+0000 7fa401ffb640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 info.flags=0x17 2026-03-20T11:46:56.462 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.465+0000 7fa401ffb640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 to cache LRU end 2026-03-20T11:46:56.462 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.465+0000 7fa401ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.462 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.465+0000 7fa401ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.462 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.465+0000 7fa401ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:56.462 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.465+0000 7fa401ffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-15 : miss 2026-03-20T11:46:56.462 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.465+0000 7fa401ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.462 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.465+0000 7fa401ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.462 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.465+0000 7fa401ffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-15 info.flags=0x0 2026-03-20T11:46:56.462 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.465+0000 7fa401ffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-15 to cache LRU end 2026-03-20T11:46:56.462 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.465+0000 7fa401ffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-15 : miss 2026-03-20T11:46:56.462 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.465+0000 7fa401ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.462 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.465+0000 7fa401ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.462 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.465+0000 7fa401ffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-15 info.flags=0x0 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.465+0000 7fa401ffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-15 to cache LRU end 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa401ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding source pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa401ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding dest pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa401ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa401ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa401ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07866420:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 20 run: stack=0x558d07856e80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07854920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07854920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = ec320f1303f8560a7e077444781e62c1a78a81035edecce27cb99cfa8c7a0ec3 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:ec320f1303f8560a7e077444781e62c1a78a81035edecce27cb99cfa8c7a0ec3 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = d6225ddea14628ecf4e5b5973975389886adc152f10a9dbd5095f805eff342ef 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=d6225ddea14628ecf4e5b5973975389886adc152f10a9dbd5095f805eff342ef 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=42, curl_handle=0x558d07868640 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa44df93900 20 run: stack=0x558d078c7b50 is io blocked 2026-03-20T11:46:56.463 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.466+0000 7fa3f0ff9640 20 link_request req_data=0x558d073d4c70 req_data->id=42, curl_handle=0x558d07868640 2026-03-20T11:46:56.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d07855010:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d07855010:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d076652b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa44df93900 15 stack 0x558d07372f40 end 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa44df93900 20 run: stack=0x558d07372f40 is done 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa44df93900 20 collect(): s=0x558d07860730 stack=0x558d07372f40 is complete 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa44df93900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa44df93900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa44df93900 15 stack 0x558d07860730 end 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa44df93900 20 stack->operate() returned ret=-22 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa44df93900 20 run: stack=0x558d07860730 is done 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa44df93900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa44df93900 4 starting trim on bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa44df93900 20 enqueued request req=0x558d07859470 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa44df93900 20 req: 0x558d07859470 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa44df93900 20 run: stack=0x558d07860730 is io blocked 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa4017fa640 20 dequeued request req=0x558d07859470 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa4017fa640 20 RGWWQ: empty 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa4017fa640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 : miss 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa4017fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa4017fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa4017fa640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 info.flags=0x17 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa4017fa640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 to cache LRU end 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa4017fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa4017fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa4017fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa4017fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-13 : miss 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa4017fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa4017fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa4017fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-13 info.flags=0x0 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa4017fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-13 to cache LRU end 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa4017fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-13 : miss 2026-03-20T11:46:56.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.470+0000 7fa4017fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa4017fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa4017fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-13 info.flags=0x0 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa4017fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-13 to cache LRU end 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa4017fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18): adding source pipe: {s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa4017fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18): adding dest pipe: {s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa4017fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa4017fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa4017fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d073d79b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d076652b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 20 run: stack=0x558d07860730 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d07855010:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d07855010:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.468 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.469 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.469 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.469 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.469 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.469 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.469 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.469 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 9568c681de0b96b4ebf9d9d85f3a06836100a2b2ee130ec479adc35d1de27bd5 2026-03-20T11:46:56.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.469 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.469 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.469 INFO:teuthology.orchestra.run.vm02.stderr:9568c681de0b96b4ebf9d9d85f3a06836100a2b2ee130ec479adc35d1de27bd5 2026-03-20T11:46:56.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = df12329d1da44bfb94e3f0381478a60b9a84a4c343fca3069630166dcfe00bf3 2026-03-20T11:46:56.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=df12329d1da44bfb94e3f0381478a60b9a84a4c343fca3069630166dcfe00bf3 2026-03-20T11:46:56.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=43, curl_handle=0x558d07864550 2026-03-20T11:46:56.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa44df93900 20 run: stack=0x558d07372f40 is io blocked 2026-03-20T11:46:56.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.471+0000 7fa3f0ff9640 20 link_request req_data=0x558d078a55e0 req_data->id=43, curl_handle=0x558d07864550 2026-03-20T11:46:56.491 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.495+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078cb1f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.495+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078cb1f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.495+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078cb1f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.495+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078cb1f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.495+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d076626a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.495+0000 7fa44df93900 15 stack 0x558d0786aaf0 end 2026-03-20T11:46:56.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.495+0000 7fa44df93900 20 run: stack=0x558d0786aaf0 is done 2026-03-20T11:46:56.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.495+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.495+0000 7fa44df93900 20 collect(): s=0x558d0789c890 stack=0x558d0786aaf0 is complete 2026-03-20T11:46:56.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.495+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.495+0000 7fa44df93900 10 trimming bilogs for bucket=:dipepu-9[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14]) markers=[,00000000002.17.6,,,,,,,,,], shards=11 2026-03-20T11:46:56.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.495+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d076626a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.495+0000 7fa44df93900 10 trimming bilog shard 1 of :dipepu-9[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14]) at marker 00000000002.17.6 2026-03-20T11:46:56.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.495+0000 7fa44df93900 20 run: stack=0x558d0789c890 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.495+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d076674c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.495+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d076674c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.495+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14.1 2026-03-20T11:46:56.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.495+0000 7fa44df93900 20 run: stack=0x558d0786aaf0 is io blocked 2026-03-20T11:46:56.492 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078c6e80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078c6e80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078c6e80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078c6e80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078a07e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 15 stack 0x558d07876860 end 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 run: stack=0x558d07876860 is done 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 collect(): s=0x558d0785c1a0 stack=0x558d07876860 is complete 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 10 trimming bilogs for bucket=:dipepu-38[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43]) markers=[,,,,,,,,00000000001.55.6,,], shards=11 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d07666b50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 10 trimming bilog shard 8 of :dipepu-38[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43]) at marker 00000000001.55.6 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078c2e70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078c2e70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43.8 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 run: stack=0x558d07876860 is io blocked 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d076674c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d076674c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 15 stack 0x558d0786aaf0 end 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 run: stack=0x558d0786aaf0 is done 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d076626a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 trim: trimmed bucket instance dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 15 stack 0x558d0789c890 end 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 run: stack=0x558d0789c890 is done 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 4 starting trim on bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d076674c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.493 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d076674c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 enqueued request req=0x558d07863d20 2026-03-20T11:46:56.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 req: 0x558d07863d20 2026-03-20T11:46:56.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa44df93900 20 run: stack=0x558d0789c890 is io blocked 2026-03-20T11:46:56.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa400ff9640 20 dequeued request req=0x558d07863d20 2026-03-20T11:46:56.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa400ff9640 20 RGWWQ: empty 2026-03-20T11:46:56.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa400ff9640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 : miss 2026-03-20T11:46:56.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.496+0000 7fa400ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.497+0000 7fa400ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.497+0000 7fa400ff9640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 info.flags=0x17 2026-03-20T11:46:56.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.497+0000 7fa400ff9640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 to cache LRU end 2026-03-20T11:46:56.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.497+0000 7fa400ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.497+0000 7fa400ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.497+0000 7fa400ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:56.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.497+0000 7fa400ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-11 : miss 2026-03-20T11:46:56.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.497+0000 7fa400ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.497+0000 7fa400ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.497+0000 7fa400ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-11 info.flags=0x0 2026-03-20T11:46:56.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.497+0000 7fa400ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-11 to cache LRU end 2026-03-20T11:46:56.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.497+0000 7fa400ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-11 : miss 2026-03-20T11:46:56.494 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.497+0000 7fa400ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078c2e70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078c2e70:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 15 stack 0x558d07876860 end 2026-03-20T11:46:56.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 run: stack=0x558d07876860 is done 2026-03-20T11:46:56.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d07666b50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 trim: trimmed bucket instance dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:46:56.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 15 stack 0x558d0785c1a0 end 2026-03-20T11:46:56.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is done 2026-03-20T11:46:56.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 4 starting trim on bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:56.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d07666b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d07666b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 enqueued request req=0x558d073db6d0 2026-03-20T11:46:56.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 req: 0x558d073db6d0 2026-03-20T11:46:56.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 run: stack=0x558d07876860 is io blocked 2026-03-20T11:46:56.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa4007f8640 20 dequeued request req=0x558d073db6d0 2026-03-20T11:46:56.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa4007f8640 20 RGWWQ: empty 2026-03-20T11:46:56.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa4007f8640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 : miss 2026-03-20T11:46:56.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa4007f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa400ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa400ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-11 info.flags=0x0 2026-03-20T11:46:56.495 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa400ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-11 to cache LRU end 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa400ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16): adding source pipe: {s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa400ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16): adding dest pipe: {s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa400ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa400ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa400ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa4007f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d076674c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d076674c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d076674c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d076674c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa4007f8640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 info.flags=0x17 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa4007f8640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 to cache LRU end 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa4007f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa4007f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d076626a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa4007f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 run: stack=0x558d0789c890 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078cb1f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078cb1f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa4007f8640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-12 : miss 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa4007f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 9468e0110782f784c279bb1e455c32042e67925e634105eef600d5c5a0ffa908 2026-03-20T11:46:56.496 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:9468e0110782f784c279bb1e455c32042e67925e634105eef600d5c5a0ffa908 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = e899a4c27d5d855d43c990f3cf0092186e0cacc1fb031277036c4b29082a88ad 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=e899a4c27d5d855d43c990f3cf0092186e0cacc1fb031277036c4b29082a88ad 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=44, curl_handle=0x558d0785fe10 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa44df93900 20 run: stack=0x558d0786aaf0 is io blocked 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa3f0ff9640 20 link_request req_data=0x558d073daf10 req_data->id=44, curl_handle=0x558d0785fe10 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa4007f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa4007f8640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-12 info.flags=0x0 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa4007f8640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-12 to cache LRU end 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa4007f8640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-12 : miss 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.498+0000 7fa4007f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa4007f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa4007f8640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-12 info.flags=0x0 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa4007f8640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-12 to cache LRU end 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa4007f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17): adding source pipe: {s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa4007f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17): adding dest pipe: {s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa4007f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa4007f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa4007f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d07666b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d07666b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d07666b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d07666b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d07871930:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 20 run: stack=0x558d07876860 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d07663f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d07663f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = b188612e1000535e20d98751357feeb55be83476a6630cb53d693351dd5270ef 2026-03-20T11:46:56.497 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.498 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.498 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.498 INFO:teuthology.orchestra.run.vm02.stderr:b188612e1000535e20d98751357feeb55be83476a6630cb53d693351dd5270ef 2026-03-20T11:46:56.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 2b49a4adecccc5ca30ba97c923d5956a2615293f9294216c98276a89a7686121 2026-03-20T11:46:56.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2b49a4adecccc5ca30ba97c923d5956a2615293f9294216c98276a89a7686121 2026-03-20T11:46:56.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=45, curl_handle=0x558d0786c860 2026-03-20T11:46:56.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa44df93900 20 run: stack=0x558d078599d0 is io blocked 2026-03-20T11:46:56.498 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.499+0000 7fa3f0ff9640 20 link_request req_data=0x558d07868b70 req_data->id=45, curl_handle=0x558d0786c860 2026-03-20T11:46:56.504 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.508+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07854920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.505 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.508+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07854920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.505 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.508+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07854920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.505 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.508+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07854920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.505 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.508+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07866420:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.505 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.508+0000 7fa44df93900 15 stack 0x558d078c7b50 end 2026-03-20T11:46:56.505 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.508+0000 7fa44df93900 20 run: stack=0x558d078c7b50 is done 2026-03-20T11:46:56.505 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.508+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.505 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.508+0000 7fa44df93900 20 collect(): s=0x558d07856e80 stack=0x558d078c7b50 is complete 2026-03-20T11:46:56.505 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.508+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.505 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.508+0000 7fa44df93900 10 trimming bilogs for bucket=:dipepu-15[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20]) markers=[,,,,,,,,00000000007.15.6,,], shards=11 2026-03-20T11:46:56.505 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.508+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d07666b50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.505 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.508+0000 7fa44df93900 10 trimming bilog shard 8 of :dipepu-15[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20]) at marker 00000000007.15.6 2026-03-20T11:46:56.505 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.508+0000 7fa44df93900 20 run: stack=0x558d07856e80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.505 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.508+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d076674c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.505 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.508+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d076674c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.505 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.508+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20.8 2026-03-20T11:46:56.505 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.508+0000 7fa44df93900 20 run: stack=0x558d078c7b50 is io blocked 2026-03-20T11:46:56.506 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.509+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d076674c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.506 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.509+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d076674c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.506 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.509+0000 7fa44df93900 15 stack 0x558d078c7b50 end 2026-03-20T11:46:56.506 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.509+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.506 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.509+0000 7fa44df93900 20 run: stack=0x558d078c7b50 is done 2026-03-20T11:46:56.506 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.509+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d07666b50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.506 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.509+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.506 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.509+0000 7fa44df93900 20 trim: trimmed bucket instance dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:46:56.506 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.509+0000 7fa44df93900 15 stack 0x558d07856e80 end 2026-03-20T11:46:56.506 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.509+0000 7fa44df93900 20 run: stack=0x558d07856e80 is done 2026-03-20T11:46:56.506 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.509+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.506 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.509+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.506 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.509+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.506 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.509+0000 7fa44df93900 4 starting trim on bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:56.506 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.509+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078c4330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.506 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.509+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078c4330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.506 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.509+0000 7fa44df93900 20 enqueued request req=0x558d07876670 2026-03-20T11:46:56.506 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.509+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.506 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.509+0000 7fa44df93900 20 req: 0x558d07876670 2026-03-20T11:46:56.506 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.509+0000 7fa44df93900 20 run: stack=0x558d07856e80 is io blocked 2026-03-20T11:46:56.506 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.509+0000 7fa3ffff7640 20 dequeued request req=0x558d07876670 2026-03-20T11:46:56.506 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.509+0000 7fa3ffff7640 20 RGWWQ: empty 2026-03-20T11:46:56.506 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.509+0000 7fa3ffff7640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 : miss 2026-03-20T11:46:56.506 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.509+0000 7fa3ffff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.507 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.510+0000 7fa3ffff7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:46:56.507 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.510+0000 7fa3ffff7640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 info.flags=0x17 2026-03-20T11:46:56.507 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.510+0000 7fa3ffff7640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 to cache LRU end 2026-03-20T11:46:56.507 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.510+0000 7fa3ffff7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.507 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.510+0000 7fa3ffff7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.507 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.510+0000 7fa3ffff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:56.507 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.510+0000 7fa3ffff7640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-40 : miss 2026-03-20T11:46:56.507 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.510+0000 7fa3ffff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.507 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.510+0000 7fa3ffff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.507 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.510+0000 7fa3ffff7640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-40 info.flags=0x0 2026-03-20T11:46:56.507 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.510+0000 7fa3ffff7640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-40 to cache LRU end 2026-03-20T11:46:56.507 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.510+0000 7fa3ffff7640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-40 : miss 2026-03-20T11:46:56.507 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.510+0000 7fa3ffff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.507 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.510+0000 7fa3ffff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.507 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.510+0000 7fa3ffff7640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-40 info.flags=0x0 2026-03-20T11:46:56.507 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.510+0000 7fa3ffff7640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-40 to cache LRU end 2026-03-20T11:46:56.507 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.510+0000 7fa3ffff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1): adding source pipe: {s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.507 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.510+0000 7fa3ffff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1): adding dest pipe: {s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.507 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.510+0000 7fa3ffff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.507 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.510+0000 7fa3ffff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.507 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.510+0000 7fa3ffff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078c4330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078c4330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078c4330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078c4330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d078a07e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 20 run: stack=0x558d07856e80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07854920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07854920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = dace42f2dc55cd1ad26863ecfc0e566386a1fa52c5e20da899905215a19ec627 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:dace42f2dc55cd1ad26863ecfc0e566386a1fa52c5e20da899905215a19ec627 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = a8c9eb9e8831a351b41ced3541c7657a217f093b190b85cd4a963d256ed64119 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a8c9eb9e8831a351b41ced3541c7657a217f093b190b85cd4a963d256ed64119 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=46, curl_handle=0x558d07868640 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa44df93900 20 run: stack=0x558d078c7b50 is io blocked 2026-03-20T11:46:56.508 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.511+0000 7fa3f0ff9640 20 link_request req_data=0x558d073d4c70 req_data->id=46, curl_handle=0x558d07868640 2026-03-20T11:46:56.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.514+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d07855010:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.514+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d07855010:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.514+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d07855010:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.514+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d07855010:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.514+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d076652b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.514+0000 7fa44df93900 15 stack 0x558d07372f40 end 2026-03-20T11:46:56.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.514+0000 7fa44df93900 20 run: stack=0x558d07372f40 is done 2026-03-20T11:46:56.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.514+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.514+0000 7fa44df93900 20 collect(): s=0x558d07860730 stack=0x558d07372f40 is complete 2026-03-20T11:46:56.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.514+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.514+0000 7fa44df93900 10 trimming bilogs for bucket=:dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) markers=[,,,,,00000000002.36.11,,,00000000011.25.6,00000000003.20.6,], shards=11 2026-03-20T11:46:56.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.514+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d07855010:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.514+0000 7fa44df93900 10 trimming bilog shard 5 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000002.36.11 2026-03-20T11:46:56.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.514+0000 7fa44df93900 10 trimming bilog shard 8 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000011.25.6 2026-03-20T11:46:56.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.514+0000 7fa44df93900 10 trimming bilog shard 9 of :dipepu-13[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18]) at marker 00000000003.20.6 2026-03-20T11:46:56.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.514+0000 7fa44df93900 20 run: stack=0x558d07860730 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.514+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078cba70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.514+0000 7fa44df93900 20 cr:s=0x558d07667cc0:op=0x558d078c6530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.514+0000 7fa44df93900 20 cr:s=0x558d073dc480:op=0x558d078c6d30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.514+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078cba70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.514+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.5 2026-03-20T11:46:56.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.514+0000 7fa44df93900 20 run: stack=0x558d07372f40 is io blocked 2026-03-20T11:46:56.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.514+0000 7fa44df93900 20 cr:s=0x558d07667cc0:op=0x558d078c6530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.514+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.8 2026-03-20T11:46:56.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.514+0000 7fa44df93900 20 run: stack=0x558d07667cc0 is io blocked 2026-03-20T11:46:56.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.514+0000 7fa44df93900 20 cr:s=0x558d073dc480:op=0x558d078c6d30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.514+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18.9 2026-03-20T11:46:56.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.514+0000 7fa44df93900 20 run: stack=0x558d073dc480 is io blocked 2026-03-20T11:46:56.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078cba70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d078cba70:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 15 stack 0x558d07372f40 end 2026-03-20T11:46:56.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 run: stack=0x558d07372f40 is done 2026-03-20T11:46:56.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d07855010:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 run: stack=0x558d07860730 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 cr:s=0x558d07667cc0:op=0x558d078c6530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 cr:s=0x558d07667cc0:op=0x558d078c6530:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 15 stack 0x558d07667cc0 end 2026-03-20T11:46:56.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 run: stack=0x558d07667cc0 is done 2026-03-20T11:46:56.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d07855010:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 run: stack=0x558d07860730 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 cr:s=0x558d073dc480:op=0x558d078c6d30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 cr:s=0x558d073dc480:op=0x558d078c6d30:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 15 stack 0x558d073dc480 end 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 run: stack=0x558d073dc480 is done 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d07855010:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 trim: trimmed bucket instance dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 15 stack 0x558d07860730 end 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 run: stack=0x558d07860730 is done 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 4 starting trim on bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d078c4330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d078c4330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 enqueued request req=0x558d078c7900 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 req: 0x558d078c7900 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa44df93900 20 run: stack=0x558d07860730 is io blocked 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa3ff7f6640 20 dequeued request req=0x558d078c7900 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa3ff7f6640 20 RGWWQ: empty 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa3ff7f6640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 : miss 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.515+0000 7fa3ff7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa3ff7f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa3ff7f6640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 info.flags=0x17 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa3ff7f6640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 to cache LRU end 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa3ff7f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa3ff7f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa3ff7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa3ff7f6640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-18 : miss 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa3ff7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa3ff7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa3ff7f6640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-18 info.flags=0x0 2026-03-20T11:46:56.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa3ff7f6640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-18 to cache LRU end 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa3ff7f6640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-18 : miss 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa3ff7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa3ff7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa3ff7f6640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-18 info.flags=0x0 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa3ff7f6640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-18 to cache LRU end 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa3ff7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23): adding source pipe: {s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa3ff7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23): adding dest pipe: {s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa3ff7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa3ff7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa3ff7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d078c4330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d078c4330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d078c4330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d078c4330:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 20 cr:s=0x558d073dc480:op=0x558d073dbb50:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 20 run: stack=0x558d07860730 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 20 cr:s=0x558d073dc480:op=0x558d07665240:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 20 cr:s=0x558d073dc480:op=0x558d07665240:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = a109de62a3dbde1a65e5745c8d4e3d8707c492441fcf605411409442afb962e9 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:a109de62a3dbde1a65e5745c8d4e3d8707c492441fcf605411409442afb962e9 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = c464bbcd62f0c74244f0dda8f651d91396eb616ce8b0df84a1be8183fea745cf 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c464bbcd62f0c74244f0dda8f651d91396eb616ce8b0df84a1be8183fea745cf 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.515 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=47, curl_handle=0x558d07864550 2026-03-20T11:46:56.515 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa44df93900 20 run: stack=0x558d073dc480 is io blocked 2026-03-20T11:46:56.515 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.516+0000 7fa3f0ff9640 20 link_request req_data=0x558d078a55e0 req_data->id=47, curl_handle=0x558d07864550 2026-03-20T11:46:56.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078cb1f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078cb1f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d07663f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078cb1f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d07663f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d078cb1f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d07663f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d076626a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.539 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 15 stack 0x558d0786aaf0 end 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 run: stack=0x558d0786aaf0 is done 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d07663f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 collect(): s=0x558d0789c890 stack=0x558d0786aaf0 is complete 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 10 trimming bilogs for bucket=:dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) markers=[00000000014.53.6,00000000008.38.6,00000000008.39.6,00000000008.35.6,00000000012.28.6,00000000010.38.6,00000000008.22.6,00000000010.27.6,00000000004.28.6,00000000008.20.6,00000000008.23.6], shards=11 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d07871930:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 15 stack 0x558d078599d0 end 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 run: stack=0x558d078599d0 is done 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d076626a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 10 trimming bilog shard 0 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000014.53.6 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 10 trimming bilog shard 1 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.38.6 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 10 trimming bilog shard 2 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.39.6 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 10 trimming bilog shard 3 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.35.6 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 10 trimming bilog shard 4 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000012.28.6 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 10 trimming bilog shard 5 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000010.38.6 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 10 trimming bilog shard 6 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.22.6 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 10 trimming bilog shard 7 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000010.27.6 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 10 trimming bilog shard 8 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000004.28.6 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 10 trimming bilog shard 9 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.20.6 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 10 trimming bilog shard 10 of :dipepu-11[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16]) at marker 00000000008.23.6 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 run: stack=0x558d0789c890 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 collect(): s=0x558d07876860 stack=0x558d078599d0 is complete 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 10 trimming bilogs for bucket=:dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) markers=[00000000014.49.6,00000000008.48.6,00000000008.43.6,00000000008.23.6,00000000012.30.6,00000000010.25.6,00000000008.36.6,00000000010.29.6,00000000004.42.6,00000000008.28.6,00000000008.26.6], shards=11 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d07866d80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d07667cc0:op=0x558d073d9f80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d07873480:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d07872a40:op=0x558d078cbb90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d073dadd0:op=0x558d07665930:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078c97b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d078c3d80:op=0x558d078c6560:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d078c9fb0:op=0x558d078cca60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d078c8810:op=0x558d078cd960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d0785a670:op=0x558d078ce160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d07666270:op=0x558d078ce960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d07871930:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 10 trimming bilog shard 0 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000014.49.6 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 10 trimming bilog shard 1 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.48.6 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 10 trimming bilog shard 2 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.43.6 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 10 trimming bilog shard 3 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.23.6 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 10 trimming bilog shard 4 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000012.30.6 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 10 trimming bilog shard 5 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000010.25.6 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 10 trimming bilog shard 6 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.36.6 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 10 trimming bilog shard 7 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000010.29.6 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 10 trimming bilog shard 8 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000004.42.6 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 10 trimming bilog shard 9 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.28.6 2026-03-20T11:46:56.540 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 10 trimming bilog shard 10 of :dipepu-12[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17]) at marker 00000000008.26.6 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 run: stack=0x558d07876860 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d07866d80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.0 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 run: stack=0x558d0786aaf0 is io blocked 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d07667cc0:op=0x558d073d9f80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.1 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 run: stack=0x558d07667cc0 is io blocked 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d07873480:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.2 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 run: stack=0x558d07372f40 is io blocked 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d07872a40:op=0x558d078cbb90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.3 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 run: stack=0x558d07872a40 is io blocked 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 cr:s=0x558d073dadd0:op=0x558d07665930:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.542+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.4 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.543+0000 7fa44df93900 20 run: stack=0x558d073dadd0 is io blocked 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.543+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078c97b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.543+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.5 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.543+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is io blocked 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.543+0000 7fa44df93900 20 cr:s=0x558d078c3d80:op=0x558d078c6560:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.543+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.6 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.543+0000 7fa44df93900 20 run: stack=0x558d078c3d80 is io blocked 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.543+0000 7fa44df93900 20 cr:s=0x558d078c9fb0:op=0x558d078cca60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.543+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.7 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.543+0000 7fa44df93900 20 run: stack=0x558d078c9fb0 is io blocked 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.543+0000 7fa44df93900 20 cr:s=0x558d078c8810:op=0x558d078cd960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.543+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.8 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.543+0000 7fa44df93900 20 run: stack=0x558d078c8810 is io blocked 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.543+0000 7fa44df93900 20 cr:s=0x558d0785a670:op=0x558d078ce160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.543+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.9 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.543+0000 7fa44df93900 20 run: stack=0x558d0785a670 is io blocked 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.544+0000 7fa44df93900 20 cr:s=0x558d07666270:op=0x558d078ce960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.541 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.544+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16.10 2026-03-20T11:46:56.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.544+0000 7fa44df93900 20 run: stack=0x558d07666270 is io blocked 2026-03-20T11:46:56.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.544+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d078cf160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.544+0000 7fa44df93900 20 cr:s=0x558d07667e50:op=0x558d078cf960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.544+0000 7fa44df93900 20 cr:s=0x558d07874420:op=0x558d078d0160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.544+0000 7fa44df93900 20 cr:s=0x558d07660f90:op=0x558d078d0960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.544+0000 7fa44df93900 20 cr:s=0x558d078ca250:op=0x558d078d1160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.544+0000 7fa44df93900 20 cr:s=0x558d0785a850:op=0x558d078d1960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.544+0000 7fa44df93900 20 cr:s=0x558d078cb8a0:op=0x558d078f4a90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.544+0000 7fa44df93900 20 cr:s=0x558d073dd1e0:op=0x558d078f5290:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.544+0000 7fa44df93900 20 cr:s=0x558d078a6c70:op=0x558d078f5a90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.544+0000 7fa44df93900 20 cr:s=0x558d078c1660:op=0x558d078f6290:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.544+0000 7fa44df93900 20 cr:s=0x558d073d8cc0:op=0x558d078f6a90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.544+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d078cf160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.544+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.0 2026-03-20T11:46:56.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.544+0000 7fa44df93900 20 run: stack=0x558d078599d0 is io blocked 2026-03-20T11:46:56.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.544+0000 7fa44df93900 20 cr:s=0x558d07667e50:op=0x558d078cf960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.544+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.1 2026-03-20T11:46:56.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.544+0000 7fa44df93900 20 run: stack=0x558d07667e50 is io blocked 2026-03-20T11:46:56.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.544+0000 7fa44df93900 20 cr:s=0x558d07874420:op=0x558d078d0160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.544+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.2 2026-03-20T11:46:56.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.544+0000 7fa44df93900 20 run: stack=0x558d07874420 is io blocked 2026-03-20T11:46:56.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.544+0000 7fa44df93900 20 cr:s=0x558d07660f90:op=0x558d078d0960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.544+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.3 2026-03-20T11:46:56.542 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.544+0000 7fa44df93900 20 run: stack=0x558d07660f90 is io blocked 2026-03-20T11:46:56.543 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.544+0000 7fa44df93900 20 cr:s=0x558d078ca250:op=0x558d078d1160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.543 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.544+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.4 2026-03-20T11:46:56.543 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.545+0000 7fa44df93900 20 run: stack=0x558d078ca250 is io blocked 2026-03-20T11:46:56.543 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.545+0000 7fa44df93900 20 cr:s=0x558d0785a850:op=0x558d078d1960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.543 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.545+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.5 2026-03-20T11:46:56.543 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.545+0000 7fa44df93900 20 run: stack=0x558d0785a850 is io blocked 2026-03-20T11:46:56.543 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.546+0000 7fa44df93900 20 cr:s=0x558d078cb8a0:op=0x558d078f4a90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.543 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.546+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.6 2026-03-20T11:46:56.543 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.546+0000 7fa44df93900 20 run: stack=0x558d078cb8a0 is io blocked 2026-03-20T11:46:56.543 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.546+0000 7fa44df93900 20 cr:s=0x558d073dd1e0:op=0x558d078f5290:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.543 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.546+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.7 2026-03-20T11:46:56.543 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d073dd1e0 is io blocked 2026-03-20T11:46:56.543 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d078a6c70:op=0x558d078f5a90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.543 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.8 2026-03-20T11:46:56.543 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d078a6c70 is io blocked 2026-03-20T11:46:56.543 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d078c1660:op=0x558d078f6290:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.543 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.9 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d078c1660 is io blocked 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d073d8cc0:op=0x558d078f6a90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17.10 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d073d8cc0 is io blocked 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d07872a40:op=0x558d078cbb90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d07872a40:op=0x558d078cbb90:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 15 stack 0x558d07872a40 end 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d07872a40 is done 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d07866d80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d0786aaf0:op=0x558d07866d80:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 15 stack 0x558d0786aaf0 end 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d0786aaf0 is done 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d073dadd0:op=0x558d07665930:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d073dadd0:op=0x558d07665930:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 15 stack 0x558d073dadd0 end 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d073dadd0 is done 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d07667cc0:op=0x558d073d9f80:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d07667cc0:op=0x558d073d9f80:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 15 stack 0x558d07667cc0 end 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d07667cc0 is done 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078c97b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d0785c1a0:op=0x558d078c97b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 15 stack 0x558d0785c1a0 end 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.544 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d0785c1a0 is done 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d0785a670:op=0x558d078ce160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d0785a670:op=0x558d078ce160:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 15 stack 0x558d0785a670 end 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d0785a670 is done 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d078c9fb0:op=0x558d078cca60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d078c9fb0:op=0x558d078cca60:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 15 stack 0x558d078c9fb0 end 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d078c9fb0 is done 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d078c8810:op=0x558d078cd960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d078c8810:op=0x558d078cd960:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 15 stack 0x558d078c8810 end 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d078c8810 is done 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d078ca250:op=0x558d078d1160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d078ca250:op=0x558d078d1160:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 15 stack 0x558d078ca250 end 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d078ca250 is done 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d078c3d80:op=0x558d078c6560:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d078c3d80:op=0x558d078c6560:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 15 stack 0x558d078c3d80 end 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d078c3d80 is done 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d07666270:op=0x558d078ce960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d07666270:op=0x558d078ce960:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 15 stack 0x558d07666270 end 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d07666270 is done 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d07660f90:op=0x558d078d0960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d07660f90:op=0x558d078d0960:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 15 stack 0x558d07660f90 end 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d07660f90 is done 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d0785a850:op=0x558d078d1960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d0785a850:op=0x558d078d1960:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 15 stack 0x558d0785a850 end 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d0785a850 is done 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d076626a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d07871930:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d076626a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d07871930:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d076626a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d07871930:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d07876860 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d076626a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d076626a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d076626a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d076626a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d076626a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d076626a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d076626a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.545 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d0789c890 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.546 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d07873480:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.546 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d07372f40:op=0x558d07873480:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.546 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 15 stack 0x558d07372f40 end 2026-03-20T11:46:56.546 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.546 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d07372f40 is done 2026-03-20T11:46:56.546 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d076626a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.546 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d078cf160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.546 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d078599d0:op=0x558d078cf160:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.546 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 15 stack 0x558d078599d0 end 2026-03-20T11:46:56.546 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.546 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d078599d0 is done 2026-03-20T11:46:56.546 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d0789c890:op=0x558d078c4aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.546 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 trim: trimmed bucket instance dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:46:56.546 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 15 stack 0x558d0789c890 end 2026-03-20T11:46:56.546 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d0789c890 is done 2026-03-20T11:46:56.546 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d07871930:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.546 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d07876860 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.546 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.546 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.546 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d073dadd0:op=0x558d078c4aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.546 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 4 starting trim on bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:56.546 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d073dadd0:op=0x558d078cca60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.546 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d073dadd0:op=0x558d078cca60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 enqueued request req=0x558d078c8360 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 req: 0x558d078c8360 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d073dadd0 is io blocked 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa3feff5640 20 dequeued request req=0x558d078c8360 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa3feff5640 20 RGWWQ: empty 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa3feff5640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 : miss 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa3feff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d078cb8a0:op=0x558d078f4a90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d078cb8a0:op=0x558d078f4a90:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 15 stack 0x558d078cb8a0 end 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d078cb8a0 is done 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d07871930:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d07876860 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d07667e50:op=0x558d078cf960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d07667e50:op=0x558d078cf960:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 15 stack 0x558d07667e50 end 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d07667e50 is done 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d07871930:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.547+0000 7fa44df93900 20 run: stack=0x558d07876860 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.548+0000 7fa44df93900 20 cr:s=0x558d07874420:op=0x558d078d0160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.548+0000 7fa44df93900 20 cr:s=0x558d07874420:op=0x558d078d0160:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.548+0000 7fa44df93900 15 stack 0x558d07874420 end 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.548+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.548+0000 7fa44df93900 20 run: stack=0x558d07874420 is done 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.548+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d07871930:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.548+0000 7fa44df93900 20 run: stack=0x558d07876860 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.548+0000 7fa44df93900 20 cr:s=0x558d073dd1e0:op=0x558d078f5290:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.548+0000 7fa44df93900 20 cr:s=0x558d073dd1e0:op=0x558d078f5290:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.548+0000 7fa44df93900 15 stack 0x558d073dd1e0 end 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.548+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.548+0000 7fa44df93900 20 run: stack=0x558d073dd1e0 is done 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.548+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d07871930:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.548+0000 7fa44df93900 20 run: stack=0x558d07876860 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.548+0000 7fa3feff5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.548+0000 7fa3feff5640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 info.flags=0x17 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.548+0000 7fa3feff5640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 to cache LRU end 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.548+0000 7fa3feff5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.548+0000 7fa3feff5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.548+0000 7fa3feff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.548+0000 7fa3feff5640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-50 : miss 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.548+0000 7fa3feff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 cr:s=0x558d078c1660:op=0x558d078f6290:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 cr:s=0x558d078c1660:op=0x558d078f6290:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 15 stack 0x558d078c1660 end 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 run: stack=0x558d078c1660 is done 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d07871930:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 run: stack=0x558d07876860 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3feff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.547 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3feff5640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-50 info.flags=0x0 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3feff5640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-50 to cache LRU end 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3feff5640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-50 : miss 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3feff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 cr:s=0x558d073d8cc0:op=0x558d078f6a90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 cr:s=0x558d073d8cc0:op=0x558d078f6a90:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 15 stack 0x558d073d8cc0 end 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 run: stack=0x558d073d8cc0 is done 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d07871930:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 run: stack=0x558d07876860 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 cr:s=0x558d078a6c70:op=0x558d078f5a90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 cr:s=0x558d078a6c70:op=0x558d078f5a90:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 15 stack 0x558d078a6c70 end 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 run: stack=0x558d078a6c70 is done 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d07871930:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 cr:s=0x558d07876860:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 trim: trimmed bucket instance dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 15 stack 0x558d07876860 end 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 run: stack=0x558d07876860 is done 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 cr:s=0x558d078cb8a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 4 starting trim on bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 cr:s=0x558d078cb8a0:op=0x558d078cd1b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 cr:s=0x558d078cb8a0:op=0x558d078cd1b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 enqueued request req=0x558d078629d0 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 RGWWQ: 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 req: 0x558d078629d0 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 run: stack=0x558d078cb8a0 is io blocked 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3fe7f4640 20 dequeued request req=0x558d078629d0 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3fe7f4640 20 RGWWQ: empty 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3fe7f4640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 : miss 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3fe7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3feff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3feff5640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-50 info.flags=0x0 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3feff5640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-50 to cache LRU end 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3feff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10): adding source pipe: {s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3feff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10): adding dest pipe: {s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3feff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3feff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3feff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 cr:s=0x558d073dadd0:op=0x558d078cca60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 cr:s=0x558d073dadd0:op=0x558d078cca60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 cr:s=0x558d073dadd0:op=0x558d078cca60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 cr:s=0x558d073dadd0:op=0x558d078cca60:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 cr:s=0x558d073dadd0:op=0x558d078c4aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 cr:s=0x558d0785a850:op=0x558d07663f90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 cr:s=0x558d073dadd0:op=0x558d078c4aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 run: stack=0x558d073dadd0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 cr:s=0x558d0785a850:op=0x558d078541f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 cr:s=0x558d0785a850:op=0x558d078541f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.548 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 6e8767c84aaab89795fde0e74994432e856c3a7cbdcba15c1bf030a7f9260ccc 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:6e8767c84aaab89795fde0e74994432e856c3a7cbdcba15c1bf030a7f9260ccc 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 9854b8ace05d81e2cb4b63ee882e5a721ebc399560b809202f0c8c0c1f92fd7d 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=9854b8ace05d81e2cb4b63ee882e5a721ebc399560b809202f0c8c0c1f92fd7d 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=48, curl_handle=0x558d0786c860 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa44df93900 20 run: stack=0x558d0785a850 is io blocked 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3f0ff9640 20 link_request req_data=0x558d07868b70 req_data->id=48, curl_handle=0x558d0786c860 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3fe7f4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3fe7f4640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 info.flags=0x17 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3fe7f4640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 to cache LRU end 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3fe7f4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3fe7f4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3fe7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3fe7f4640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-24 : miss 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3fe7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3fe7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3fe7f4640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-24 info.flags=0x0 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3fe7f4640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-24 to cache LRU end 2026-03-20T11:46:56.549 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3fe7f4640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-24 : miss 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.549+0000 7fa3fe7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa3fe7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa3fe7f4640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-24 info.flags=0x0 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa3fe7f4640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-24 to cache LRU end 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa3fe7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29): adding source pipe: {s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa3fe7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29): adding dest pipe: {s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa3fe7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa3fe7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa3fe7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 20 cr:s=0x558d078cb8a0:op=0x558d078cd1b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 20 cr:s=0x558d078cb8a0:op=0x558d078cd1b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 20 cr:s=0x558d078cb8a0:op=0x558d078cd1b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 20 cr:s=0x558d078cb8a0:op=0x558d078cd1b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 20 cr:s=0x558d078cb8a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 20 cr:s=0x558d07872a40:op=0x558d076626a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 20 cr:s=0x558d078cb8a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 20 run: stack=0x558d078cb8a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 20 cr:s=0x558d07872a40:op=0x558d073dca40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 20 cr:s=0x558d07872a40:op=0x558d073dca40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:46:56 +0000 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114656Z 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 424781cae52a1c722fa6993b1f3a78892fa6a61debd8fd6d62617958d3175bbe 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114656Z 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:424781cae52a1c722fa6993b1f3a78892fa6a61debd8fd6d62617958d3175bbe 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = e63e8c12d3164684b43a5db09df2d0e669c9115c4bd5311443c1e883abeb32d8 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=e63e8c12d3164684b43a5db09df2d0e669c9115c4bd5311443c1e883abeb32d8 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114656Z 2026-03-20T11:46:56.550 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:46:56.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 20 register_request mgr=0x7fffbee18ea0 req_data->id=49, curl_handle=0x558d0785fe10 2026-03-20T11:46:56.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa44df93900 20 run: stack=0x558d07872a40 is io blocked 2026-03-20T11:46:56.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.550+0000 7fa3f0ff9640 20 link_request req_data=0x558d073daf10 req_data->id=49, curl_handle=0x558d0785fe10 2026-03-20T11:46:56.551 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:46:56.551+0000 7f12e1ab3640 -1 req 17677840832995524330 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:46:56.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.554+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07854920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.554+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07854920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.554+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07854920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.554+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d07854920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.554+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d078a07e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.554+0000 7fa44df93900 15 stack 0x558d078c7b50 end 2026-03-20T11:46:56.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.554+0000 7fa44df93900 20 run: stack=0x558d078c7b50 is done 2026-03-20T11:46:56.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.554+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.554+0000 7fa44df93900 20 collect(): s=0x558d07856e80 stack=0x558d078c7b50 is complete 2026-03-20T11:46:56.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.554+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.554+0000 7fa44df93900 10 trimming bilogs for bucket=:dipepu-40[4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1]) markers=[,,,,,00000000001.144.6,,,,,], shards=11 2026-03-20T11:46:56.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.554+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d07871910:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.554+0000 7fa44df93900 10 trimming bilog shard 5 of :dipepu-40[4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1]) at marker 00000000001.144.6 2026-03-20T11:46:56.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.554+0000 7fa44df93900 20 run: stack=0x558d07856e80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.554+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d078ad160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.554+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d078ad160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.554+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1.5 2026-03-20T11:46:56.551 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.554+0000 7fa44df93900 20 run: stack=0x558d078c7b50 is io blocked 2026-03-20T11:46:56.552 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.555+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d078ad160:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.552 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.555+0000 7fa44df93900 20 cr:s=0x558d078c7b50:op=0x558d078ad160:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.552 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.555+0000 7fa44df93900 15 stack 0x558d078c7b50 end 2026-03-20T11:46:56.552 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.555+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.552 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.555+0000 7fa44df93900 20 run: stack=0x558d078c7b50 is done 2026-03-20T11:46:56.552 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.555+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d07871910:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.552 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.555+0000 7fa44df93900 20 cr:s=0x558d07856e80:op=0x558d078a7690:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.552 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.555+0000 7fa44df93900 20 trim: trimmed bucket instance dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:46:56.552 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.555+0000 7fa44df93900 15 stack 0x558d07856e80 end 2026-03-20T11:46:56.552 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.555+0000 7fa44df93900 20 run: stack=0x558d07856e80 is done 2026-03-20T11:46:56.552 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.555+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.552 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.555+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.560+0000 7fa44df93900 20 cr:s=0x558d073dc480:op=0x558d07665240:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.560+0000 7fa44df93900 20 cr:s=0x558d073dc480:op=0x558d07665240:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.560+0000 7fa44df93900 20 cr:s=0x558d073dc480:op=0x558d07665240:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.560+0000 7fa44df93900 20 cr:s=0x558d073dc480:op=0x558d07665240:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.560+0000 7fa44df93900 20 cr:s=0x558d073dc480:op=0x558d073dbb50:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.560+0000 7fa44df93900 15 stack 0x558d073dc480 end 2026-03-20T11:46:56.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.560+0000 7fa44df93900 20 run: stack=0x558d073dc480 is done 2026-03-20T11:46:56.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.560+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.560+0000 7fa44df93900 20 collect(): s=0x558d07860730 stack=0x558d073dc480 is complete 2026-03-20T11:46:56.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.560+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.560+0000 7fa44df93900 10 trimming bilogs for bucket=:dipepu-18[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23]) markers=[,,,,,,,,00000000002.59.6,,], shards=11 2026-03-20T11:46:56.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.560+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d07871910:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.560+0000 7fa44df93900 10 trimming bilog shard 8 of :dipepu-18[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23]) at marker 00000000002.59.6 2026-03-20T11:46:56.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.560+0000 7fa44df93900 20 run: stack=0x558d07860730 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.560+0000 7fa44df93900 20 cr:s=0x558d073dc480:op=0x558d07665930:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.560+0000 7fa44df93900 20 cr:s=0x558d073dc480:op=0x558d07665930:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.560+0000 7fa44df93900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23.8 2026-03-20T11:46:56.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.560+0000 7fa44df93900 20 run: stack=0x558d073dc480 is io blocked 2026-03-20T11:46:56.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.561+0000 7fa44df93900 20 cr:s=0x558d073dc480:op=0x558d07665930:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:46:56.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.561+0000 7fa44df93900 20 cr:s=0x558d073dc480:op=0x558d07665930:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:46:56.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.561+0000 7fa44df93900 15 stack 0x558d073dc480 end 2026-03-20T11:46:56.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.561+0000 7fa44df93900 20 stack->operate() returned ret=-61 2026-03-20T11:46:56.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.561+0000 7fa44df93900 20 run: stack=0x558d073dc480 is done 2026-03-20T11:46:56.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.561+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d07871910:24BucketTrimShardCollectCR: operate() 2026-03-20T11:46:56.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.561+0000 7fa44df93900 20 cr:s=0x558d07860730:op=0x558d07662cc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.561+0000 7fa44df93900 20 trim: trimmed bucket instance dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:46:56.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.561+0000 7fa44df93900 15 stack 0x558d07860730 end 2026-03-20T11:46:56.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.561+0000 7fa44df93900 20 run: stack=0x558d07860730 is done 2026-03-20T11:46:56.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.561+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.561+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 cr:s=0x558d0785a850:op=0x558d078541f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 cr:s=0x558d0785a850:op=0x558d078541f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 cr:s=0x558d07872a40:op=0x558d073dca40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 cr:s=0x558d07872a40:op=0x558d073dca40:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:46:56.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 cr:s=0x558d0785a850:op=0x558d078541f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 cr:s=0x558d07872a40:op=0x558d076626a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 15 stack 0x558d07872a40 end 2026-03-20T11:46:56.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 run: stack=0x558d07872a40 is done 2026-03-20T11:46:56.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 cr:s=0x558d0785a850:op=0x558d078541f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:46:56.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 cr:s=0x558d078cb8a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 collect(): s=0x558d078cb8a0 stack=0x558d07872a40 is complete 2026-03-20T11:46:56.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:56.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:56.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 cr:s=0x558d078cb8a0:op=0x558d078a8ad0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:56.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 15 stack 0x558d078cb8a0 end 2026-03-20T11:46:56.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 stack->operate() returned ret=-22 2026-03-20T11:46:56.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 run: stack=0x558d078cb8a0 is done 2026-03-20T11:46:56.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 cr:s=0x558d0785a850:op=0x558d07663f90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:46:56.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 15 stack 0x558d0785a850 end 2026-03-20T11:46:56.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 run: stack=0x558d0785a850 is done 2026-03-20T11:46:56.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:56.589 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 run: stack=0x558d078528a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:46:56.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 cr:s=0x558d073dadd0:op=0x558d078c4aa0:20BucketTrimInstanceCR: operate() 2026-03-20T11:46:56.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 collect(): s=0x558d073dadd0 stack=0x558d0785a850 is complete 2026-03-20T11:46:56.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 10 min_generation is 0 2026-03-20T11:46:56.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:46:56.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 4 failed to correlate bucket sync status from peers 2026-03-20T11:46:56.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 cr:s=0x558d073dadd0:op=0x558d078c4aa0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:46:56.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 15 stack 0x558d073dadd0 end 2026-03-20T11:46:56.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 stack->operate() returned ret=-22 2026-03-20T11:46:56.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 run: stack=0x558d073dadd0 is done 2026-03-20T11:46:56.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:46:56.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:46:56.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07856600:27BucketTrimInstanceCollectCR: operate() returned r=-22 2026-03-20T11:46:56.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07852ae0:12BucketTrimCR: operate() 2026-03-20T11:46:56.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 writing bucket trim marker=16:4ca02412:root::dipepu-46:head 2026-03-20T11:46:56.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d078a7690:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:56.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d078a7690:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:56.590 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.592+0000 7fa44df93900 20 run: stack=0x558d078528a0 is io blocked 2026-03-20T11:46:56.591 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.594+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d078a7690:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:56.591 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.594+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d078a7690:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:56.591 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.594+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d078a7690:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:56.591 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.594+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d078a7690:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:46:56.591 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.594+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07852ae0:12BucketTrimCR: operate() 2026-03-20T11:46:56.591 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.594+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d078c0e70:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:56.591 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.594+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d078c0e70:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:56.591 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.594+0000 7fa44df93900 20 run: stack=0x558d078528a0 is io blocked 2026-03-20T11:46:56.591 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.594+0000 7fa424ff9640 20 trim: bucket trim completed 2026-03-20T11:46:56.592 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.595+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d078c0e70:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:56.592 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.595+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d078c0e70:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:56.592 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.595+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d078c0e70:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:56.592 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.595+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d078c0e70:16RGWRadosNotifyCR: operate() 2026-03-20T11:46:56.592 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.595+0000 7fa44df93900 20 cr:s=0x558d078528a0:op=0x558d07852ae0:12BucketTrimCR: operate() 2026-03-20T11:46:56.592 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.595+0000 7fa44df93900 4 bucket index log processing completed in 0.724809408s 2026-03-20T11:46:56.592 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.595+0000 7fa44df93900 15 stack 0x558d078528a0 end 2026-03-20T11:46:56.592 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.595+0000 7fa44df93900 20 run: stack=0x558d078528a0 is done 2026-03-20T11:46:56.597 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.601+0000 7fa44df93900 20 remove_watcher() i=1 2026-03-20T11:46:56.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.601+0000 7fa44df93900 2 removed watcher, disabling cache 2026-03-20T11:46:56.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.601+0000 7fa44df93900 20 remove_watcher() i=0 2026-03-20T11:46:56.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.602+0000 7fa44df93900 20 remove_watcher() i=3 2026-03-20T11:46:56.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.602+0000 7fa44df93900 20 remove_watcher() i=5 2026-03-20T11:46:56.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.602+0000 7fa44df93900 20 remove_watcher() i=4 2026-03-20T11:46:56.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.602+0000 7fa44df93900 20 remove_watcher() i=6 2026-03-20T11:46:56.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.602+0000 7fa44df93900 20 remove_watcher() i=7 2026-03-20T11:46:56.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:46:56.602+0000 7fa44df93900 20 remove_watcher() i=2 2026-03-20T11:47:01.705 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:47:01.705 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:47:01.705 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:47:01.745 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:47:01.746 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:47:01.843 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:47:01.857 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774007137.857782_141.1","last_update":"2026-03-20T11:45:37.857782Z"},{"marker":"1_1774007137.912792_174.1","last_update":"2026-03-20T11:45:37.912792Z"},{"marker":"1_1774007209.892139_281.1","last_update":"2026-03-20T11:46:49.892139Z"},{"marker":"1_1774007039.336409_210.1","last_update":"2026-03-20T11:43:59.336409Z"}] 2026-03-20T11:47:01.857 INFO:rgw_multi.tests:starting meta checkpoint for zone=b1 2026-03-20T11:47:01.857 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:47:01.857 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:47:01.857 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:47:01.898 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:47:01.898 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:47:01.975 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:01.977+0000 7f2184017900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:47:01.976 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:01.978+0000 7f2184017900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:47:01.987 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774007137.857782_141.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:45:37.857782Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774007137.912792_174.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:37.912792Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774007209.892139_281.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:46:49.892139Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774007039.336409_210.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:59.336409Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:47:01Z"} 2026-03-20T11:47:01.987 INFO:rgw_multi.tests:finish meta checkpoint for zone=b1 2026-03-20T11:47:01.987 INFO:rgw_multi.tests:starting meta checkpoint for zone=b2 2026-03-20T11:47:01.987 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:47:01.987 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:47:01.987 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone b2 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:47:02.070 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:47:02.070 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:47:02.140 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.142+0000 7f61bbb58900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:47:02.141 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.143+0000 7f61bbb58900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:47:02.150 INFO:teuthology.orchestra.run.vm07.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774007137.857782_141.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:45:37.857782Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774007137.912792_174.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:37.912792Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774007209.892139_281.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:46:49.892139Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774007039.336409_210.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:59.336409Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:47:02Z"} 2026-03-20T11:47:02.151 INFO:rgw_multi.tests:finish meta checkpoint for zone=b2 2026-03-20T11:47:02.151 INFO:rgw_multi.tests:trimming on zone=b1 2026-03-20T11:47:02.151 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '50', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:47:02.151 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '50', '--rgw-zone', 'b1', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:47:02.151 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bilog autotrim --rgw-sync-log-trim-max-buckets 50 --rgw-zone b1 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 20 --debug-ms 0 2026-03-20T11:47:02.230 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:47:02.231 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:47:02.248 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.250+0000 7fc30894f900 20 rados->read ofs=0 len=0 2026-03-20T11:47:02.250 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.251+0000 7fc30894f900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:47:02.250 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.251+0000 7fc30894f900 20 rados->read ofs=0 len=0 2026-03-20T11:47:02.250 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.251+0000 7fc30894f900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:47:02.250 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.251+0000 7fc30894f900 20 realm test-realm 6a789b89-c2b8-45f5-82cc-fb80234d0c6c 2026-03-20T11:47:02.250 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.251+0000 7fc30894f900 20 rados->read ofs=0 len=0 2026-03-20T11:47:02.250 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.252+0000 7fc30894f900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:47:02.250 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.252+0000 7fc30894f900 20 rados->read ofs=0 len=0 2026-03-20T11:47:02.250 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.252+0000 7fc30894f900 20 rados_obj.operate() r=0 bl.length=10 2026-03-20T11:47:02.251 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.252+0000 7fc30894f900 20 rados->read ofs=0 len=0 2026-03-20T11:47:02.251 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.253+0000 7fc30894f900 20 rados_obj.operate() r=0 bl.length=2020 2026-03-20T11:47:02.251 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.253+0000 7fc30894f900 20 rados->read ofs=0 len=0 2026-03-20T11:47:02.251 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.253+0000 7fc30894f900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:47:02.251 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.253+0000 7fc30894f900 20 rados->read ofs=0 len=0 2026-03-20T11:47:02.251 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.253+0000 7fc30894f900 20 rados_obj.operate() r=0 bl.length=1030 2026-03-20T11:47:02.251 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.253+0000 7fc30894f900 20 zone b1 found 2026-03-20T11:47:02.252 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.253+0000 7fc30894f900 4 Realm: test-realm (6a789b89-c2b8-45f5-82cc-fb80234d0c6c) 2026-03-20T11:47:02.252 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.253+0000 7fc30894f900 4 ZoneGroup: b (b) 2026-03-20T11:47:02.252 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.253+0000 7fc30894f900 4 Zone: b1 (efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c) 2026-03-20T11:47:02.252 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.253+0000 7fc30894f900 4 using period configuration: cb2fcdce-2c1b-429a-8601-a284ee3a2381:12 2026-03-20T11:47:02.252 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.253+0000 7fc30894f900 20 period zonegroup name b 2026-03-20T11:47:02.252 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.253+0000 7fc30894f900 20 using current period zonegroup b 2026-03-20T11:47:02.252 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.253+0000 7fc30894f900 20 rados->read ofs=0 len=0 2026-03-20T11:47:02.252 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.254+0000 7fc30894f900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.252 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.254+0000 7fc30894f900 20 rados->read ofs=0 len=0 2026-03-20T11:47:02.252 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.254+0000 7fc30894f900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.253 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.254+0000 7fc30894f900 20 init(): pipe_map (zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:47:02.253 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.254+0000 7fc30894f900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.253 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.254+0000 7fc30894f900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.253 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.254+0000 7fc30894f900 20 rados->read ofs=0 len=0 2026-03-20T11:47:02.253 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.255+0000 7fc30894f900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.253 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.255+0000 7fc30894f900 20 rados->read ofs=0 len=0 2026-03-20T11:47:02.253 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.255+0000 7fc30894f900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.253 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.255+0000 7fc30894f900 20 init(): pipe_map (zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:47:02.253 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.255+0000 7fc30894f900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.253 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.255+0000 7fc30894f900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.253 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.255+0000 7fc30894f900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.253 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.255+0000 7fc30894f900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.253 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.255+0000 7fc30894f900 20 started sync module instance, tier type = 2026-03-20T11:47:02.253 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.255+0000 7fc30894f900 20 generating connection object for zone b2 id cd52b229-0d4a-43bf-b4ba-156aa19e7aef 2026-03-20T11:47:02.253 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.255+0000 7fc30894f900 20 started zone id=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c (name=b1) with tier type = 2026-03-20T11:47:02.258 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.260+0000 7fc30894f900 20 add_watcher() i=5 2026-03-20T11:47:02.258 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.260+0000 7fc30894f900 20 add_watcher() i=1 2026-03-20T11:47:02.259 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.260+0000 7fc30894f900 20 add_watcher() i=6 2026-03-20T11:47:02.259 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.260+0000 7fc30894f900 20 add_watcher() i=0 2026-03-20T11:47:02.259 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.260+0000 7fc30894f900 20 add_watcher() i=2 2026-03-20T11:47:02.259 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.260+0000 7fc30894f900 20 add_watcher() i=7 2026-03-20T11:47:02.259 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.260+0000 7fc30894f900 20 add_watcher() i=3 2026-03-20T11:47:02.259 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.260+0000 7fc30894f900 20 add_watcher() i=4 2026-03-20T11:47:02.259 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.260+0000 7fc30894f900 2 all 8 watchers are set, enabling cache 2026-03-20T11:47:02.261 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.263+0000 7fc30894f900 20 rgw_check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2026-03-20T11:47:02.261 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.263+0000 7fc30894f900 20 rgw_check_secure_mon_conn(): mode 1 is insecure 2026-03-20T11:47:02.261 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.263+0000 7fc30894f900 5 note: GC not initialized 2026-03-20T11:47:02.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.315+0000 7fc30894f900 20 init_complete bucket index max shards: 11 2026-03-20T11:47:02.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.315+0000 7fc30894f900 20 Filter name: none 2026-03-20T11:47:02.312 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.315+0000 7fc2b1fdb640 20 reqs_thread_entry: start 2026-03-20T11:47:02.313 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.315+0000 7fc263fff640 20 reqs_thread_entry: start 2026-03-20T11:47:02.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.316+0000 7fc30894f900 10 rgw bucket trim manager: Watching bilog.trim 2026-03-20T11:47:02.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.316+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf8b9b0:12BucketTrimCR: operate() 2026-03-20T11:47:02.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.316+0000 7fc30894f900 10 fetching active bucket counters 2026-03-20T11:47:02.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.316+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf8e0d0:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:02.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.316+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf8e0d0:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:02.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.316+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is io blocked 2026-03-20T11:47:02.314 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.316+0000 7fc2d2ffd640 20 trim: get_bucket_counters: [] 2026-03-20T11:47:02.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.317+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf8e0d0:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:02.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.317+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf8e0d0:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:02.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.317+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf8e0d0:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:02.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.317+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf8e0d0:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:02.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.317+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf8b9b0:12BucketTrimCR: operate() 2026-03-20T11:47:02.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.317+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf8f500:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:02.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.317+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf8f500:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:02.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.317+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is io blocked 2026-03-20T11:47:02.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.317+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf8f500:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:02.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.317+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf8f500:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:02.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.317+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf8f500:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:02.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.317+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf8f500:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:02.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.317+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf8b9b0:12BucketTrimCR: operate() 2026-03-20T11:47:02.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.317+0000 7fc30894f900 10 listing cold buckets from marker=12:ca472e1e:root::.bucket.meta.dipepu-54%3a4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14:head 2026-03-20T11:47:02.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.317+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:14MetadataListCR: operate() 2026-03-20T11:47:02.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.317+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:14MetadataListCR: operate() 2026-03-20T11:47:02.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.317+0000 7fc30894f900 20 enqueued request req=0x7fc2c8002cc0 2026-03-20T11:47:02.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.317+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.317+0000 7fc30894f900 20 req: 0x7fc2c8002cc0 2026-03-20T11:47:02.316 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.317+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is io blocked 2026-03-20T11:47:02.316 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.317+0000 7fc2c37fe640 20 dequeued request req=0x7fc2c8002cc0 2026-03-20T11:47:02.316 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.317+0000 7fc2c37fe640 20 RGWWQ: empty 2026-03-20T11:47:02.316 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.317+0000 7fc2c37fe640 20 rgw async rados processor: starting metadata listing at 12:ca472e1e:root::.bucket.meta.dipepu-54%3a4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14:head 2026-03-20T11:47:02.317 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.319+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:47:02.319 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.321+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-40 2026-03-20T11:47:02.319 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.321+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-22 2026-03-20T11:47:02.319 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.321+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:47:02.320 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.322+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-56 2026-03-20T11:47:02.320 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.322+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 2026-03-20T11:47:02.321 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.323+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:47:02.323 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.325+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:47:02.324 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.326+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:47:02.325 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.327+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-53 2026-03-20T11:47:02.325 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.327+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:47:02.326 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.328+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:47:02.327 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.329+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:47:02.328 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.330+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:47:02.328 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.331+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:47:02.329 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.331+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-34 2026-03-20T11:47:02.329 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.331+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-21 2026-03-20T11:47:02.329 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.331+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-14 2026-03-20T11:47:02.329 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.331+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-50 2026-03-20T11:47:02.329 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.331+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-57 2026-03-20T11:47:02.329 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.331+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:47:02.329 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.331+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-39 2026-03-20T11:47:02.329 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.331+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:47:02.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.332+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-37 2026-03-20T11:47:02.330 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.332+0000 7fc2c37fe640 20 rgw async rados processor: restarting metadata listing 2026-03-20T11:47:02.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.338+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-10 2026-03-20T11:47:02.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.338+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:47:02.342 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.344+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:47:02.348 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.350+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:47:02.354 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.356+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:47:02.361 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.363+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:47:02.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.370+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:47:02.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.376+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-19 2026-03-20T11:47:02.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.376+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-3 2026-03-20T11:47:02.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.376+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:47:02.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.382+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:47:02.385 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.388+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-17 2026-03-20T11:47:02.386 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.388+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:47:02.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.393+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-38 2026-03-20T11:47:02.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.393+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:47:02.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.399+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-24 2026-03-20T11:47:02.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.399+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-36 2026-03-20T11:47:02.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.399+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:47:02.402 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.404+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:47:02.408 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.410+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:47:02.413 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.415+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-44 2026-03-20T11:47:02.413 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.415+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-35 2026-03-20T11:47:02.413 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.415+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:47:02.419 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.421+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-16 2026-03-20T11:47:02.419 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.421+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:47:02.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.425+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:47:02.428 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.430+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:47:02.433 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.435+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-55 2026-03-20T11:47:02.433 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.435+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-13 2026-03-20T11:47:02.433 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.435+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-48 2026-03-20T11:47:02.433 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.435+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-25 2026-03-20T11:47:02.433 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.435+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-51 2026-03-20T11:47:02.433 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.435+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:47:02.438 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.440+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:47:02.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.446+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-46 2026-03-20T11:47:02.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.446+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:47:02.448 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.450+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:47:02.453 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.455+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:47:02.457 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.459+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-26 2026-03-20T11:47:02.457 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.459+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-20 2026-03-20T11:47:02.457 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.459+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:47:02.461 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.463+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:47:02.465 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.467+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-1 2026-03-20T11:47:02.467 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.467+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:47:02.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.471+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:47:02.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.475+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:47:02.476 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.478+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-52 2026-03-20T11:47:02.476 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.478+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-32 2026-03-20T11:47:02.476 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.478+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-42 2026-03-20T11:47:02.476 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.478+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:47:02.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.482+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:47:02.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.485+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-7 2026-03-20T11:47:02.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.485+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-54 2026-03-20T11:47:02.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.485+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-49 2026-03-20T11:47:02.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.485+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-47 2026-03-20T11:47:02.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.485+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 2026-03-20T11:47:02.486 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.488+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:47:02.489 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.491+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:47:02.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.494+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-11 2026-03-20T11:47:02.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.494+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-12 2026-03-20T11:47:02.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.494+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-41 2026-03-20T11:47:02.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.494+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-23 2026-03-20T11:47:02.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.494+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-15 2026-03-20T11:47:02.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.494+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-45 2026-03-20T11:47:02.492 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.494+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:47:02.496 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.498+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:47:02.498 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.500+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-9 2026-03-20T11:47:02.498 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.500+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:47:02.501 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.503+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-29 2026-03-20T11:47:02.501 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.503+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:47:02.505 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-31 2026-03-20T11:47:02.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc2c37fe640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:47:02.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:14MetadataListCR: operate() 2026-03-20T11:47:02.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:14MetadataListCR: operate() 2026-03-20T11:47:02.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:14MetadataListCR: operate() 2026-03-20T11:47:02.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:14MetadataListCR: operate() 2026-03-20T11:47:02.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf8b9b0:12BucketTrimCR: operate() 2026-03-20T11:47:02.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 4 collected 50 buckets for trim 2026-03-20T11:47:02.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 4 starting trim on bucket=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:47:02.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf909a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 4 starting trim on bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:47:02.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 4 starting trim on bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:47:02.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 4 starting trim on bucket=dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 2026-03-20T11:47:02.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbf8f500:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf94210:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf94c30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf95650:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbf8f500:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 enqueued request req=0x5560dbf96000 2026-03-20T11:47:02.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 req: 0x5560dbf96000 2026-03-20T11:47:02.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 run: stack=0x5560dbea7100 is io blocked 2026-03-20T11:47:02.508 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf94210:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc2c2ffd640 20 dequeued request req=0x5560dbf96000 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc2c2ffd640 20 RGWWQ: empty 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 enqueued request req=0x5560dbf96420 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc2c2ffd640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 : miss 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 req: 0x5560dbf96420 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc2c2ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is io blocked 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf94c30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 enqueued request req=0x5560dbf96840 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 req: 0x5560dbf96420 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 req: 0x5560dbf96840 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is io blocked 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf95650:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc2c1ffb640 20 dequeued request req=0x5560dbf96420 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc2c1ffb640 20 RGWWQ: 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc2c1ffb640 20 req: 0x5560dbf96840 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc2c1ffb640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 : miss 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc2c1ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 enqueued request req=0x5560dbf96c60 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 req: 0x5560dbf96840 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 req: 0x5560dbf96c60 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is io blocked 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc2c17fa640 20 dequeued request req=0x5560dbf96840 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc2c17fa640 20 RGWWQ: 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc2c17fa640 20 req: 0x5560dbf96c60 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc2c17fa640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 : miss 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.506+0000 7fc2c17fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c27fc640 20 dequeued request req=0x5560dbf96c60 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c27fc640 20 RGWWQ: empty 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c27fc640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 : miss 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c27fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c2ffd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=387 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c2ffd640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 info.flags=0x17 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c2ffd640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 to cache LRU end 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c2ffd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c2ffd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c2ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c2ffd640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-61 : miss 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c2ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c27fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=386 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c27fc640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 info.flags=0x17 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c27fc640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 to cache LRU end 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c27fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c27fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c27fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c27fc640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-5 : miss 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c27fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c1ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c1ffb640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 info.flags=0x17 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c1ffb640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 to cache LRU end 2026-03-20T11:47:02.509 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c1ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c1ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c1ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:47:02.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c1ffb640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-54 : miss 2026-03-20T11:47:02.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c1ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c17fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c17fa640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 info.flags=0x17 2026-03-20T11:47:02.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c17fa640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 to cache LRU end 2026-03-20T11:47:02.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c17fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c17fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c17fa640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:47:02.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c17fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:47:02.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c17fa640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-32 : miss 2026-03-20T11:47:02.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c17fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.515 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.514+0000 7f3ed0186640 -1 req 12129489845632873818 0.005000007s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c27fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c27fc640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-5 info.flags=0x0 2026-03-20T11:47:02.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c27fc640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-5 to cache LRU end 2026-03-20T11:47:02.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c27fc640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-5 : miss 2026-03-20T11:47:02.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c27fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c1ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c1ffb640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-54 info.flags=0x0 2026-03-20T11:47:02.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c1ffb640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-54 to cache LRU end 2026-03-20T11:47:02.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c1ffb640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-54 : miss 2026-03-20T11:47:02.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c1ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c17fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c17fa640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-32 info.flags=0x0 2026-03-20T11:47:02.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c17fa640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-32 to cache LRU end 2026-03-20T11:47:02.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c17fa640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-32 : miss 2026-03-20T11:47:02.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c17fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c2ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c2ffd640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-61 info.flags=0x0 2026-03-20T11:47:02.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c2ffd640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-61 to cache LRU end 2026-03-20T11:47:02.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c2ffd640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-61 : miss 2026-03-20T11:47:02.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.507+0000 7fc2c2ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c1ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c1ffb640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-54 info.flags=0x0 2026-03-20T11:47:02.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c1ffb640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-54 to cache LRU end 2026-03-20T11:47:02.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c1ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14): adding source pipe: {s={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c1ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14): adding dest pipe: {s={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c1ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c1ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c17fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c17fa640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-32 info.flags=0x0 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c17fa640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-32 to cache LRU end 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c17fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37): adding source pipe: {s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c17fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37): adding dest pipe: {s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c17fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c17fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c17fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf94c30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c1ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf94c30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf94210:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf94c30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c27fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c27fc640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-5 info.flags=0x0 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c27fc640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-5 to cache LRU end 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c27fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10): adding source pipe: {s={b=dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c27fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10): adding dest pipe: {s={b=dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf94210:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c27fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf94c30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c27fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf94210:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c27fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c2ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c2ffd640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-61 info.flags=0x0 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c2ffd640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-61 to cache LRU end 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c2ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21): adding source pipe: {s={b=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c2ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21): adding dest pipe: {s={b=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf94210:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c2ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c2ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf95650:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc2c2ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf970c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.518 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf909a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf95650:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbf8f500:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 cr:s=0x5560dbf94e40:op=0x5560dbf97b90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf909a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf95650:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbf8f500:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 0822fc0773330ab5e1948dba16ee98e307ee3f8147a03d9dd61f729cc1347a30 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:0822fc0773330ab5e1948dba16ee98e307ee3f8147a03d9dd61f729cc1347a30 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = ce773e7cebbb7515e5cfb00bcc47a08285b381a7ab5ebc3c9eba5cecda9022c7 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=ce773e7cebbb7515e5cfb00bcc47a08285b381a7ab5ebc3c9eba5cecda9022c7 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=0, curl_handle=0x5560dbf99fc0 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.508+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is io blocked 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 cr:s=0x5560dbf94e40:op=0x5560dbf981b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc263fff640 20 link_request req_data=0x5560dbf9aa80 req_data->id=0, curl_handle=0x5560dbf99fc0 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf95650:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbf8f500:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 cr:s=0x5560dbf94e40:op=0x5560dbf981b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.519 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 36b0faf76a301c1d654aec70c58d9ff16493f177c72aa2eeae062e1df2e10948 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:36b0faf76a301c1d654aec70c58d9ff16493f177c72aa2eeae062e1df2e10948 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 28e00f3e34333cd01a31f2d2a51ca241dc4890ddb392ba37d4de30186b8528a3 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=28e00f3e34333cd01a31f2d2a51ca241dc4890ddb392ba37d4de30186b8528a3 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=1, curl_handle=0x5560dbf9d060 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 run: stack=0x5560dbf94e40 is io blocked 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc263fff640 20 link_request req_data=0x5560dbf9d760 req_data->id=1, curl_handle=0x5560dbf9d060 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbf8f500:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 cr:s=0x5560dbf9fe00:op=0x5560dbf9f700:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.520 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 cr:s=0x5560dbf9fe00:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 cr:s=0x5560dbfa0a80:op=0x5560dbfa0460:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 run: stack=0x5560dbea7100 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 cr:s=0x5560dbf9fe00:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 5a1c544d989634848098826bbaaaecc334d4116255f36f43e5eda520d3bdd943 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:5a1c544d989634848098826bbaaaecc334d4116255f36f43e5eda520d3bdd943 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = bfd17e85b94034d4eef4aa202c1bd665db16944f3b2aba43e24168c49ff9eb50 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=bfd17e85b94034d4eef4aa202c1bd665db16944f3b2aba43e24168c49ff9eb50 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=2, curl_handle=0x5560dbfa2840 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 run: stack=0x5560dbf9fe00 is io blocked 2026-03-20T11:47:02.521 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc263fff640 20 link_request req_data=0x5560dbfa2f90 req_data->id=2, curl_handle=0x5560dbfa2840 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 cr:s=0x5560dbfa0a80:op=0x5560dbfa0c10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 cr:s=0x5560dbfa0a80:op=0x5560dbfa0c10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 54a7cb54189d09f692c2012666483f9d6c567ba488be2193c085d4cdb941c8c7 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:54a7cb54189d09f692c2012666483f9d6c567ba488be2193c085d4cdb941c8c7 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.509+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.510+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.510+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = dad84cdac58a62ff3c502e710e7be2a81ce29eb86a447d600a044f83fc7a3ebc 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.510+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=dad84cdac58a62ff3c502e710e7be2a81ce29eb86a447d600a044f83fc7a3ebc 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.510+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.510+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.510+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.510+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=3, curl_handle=0x5560dbfa5a90 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.510+0000 7fc30894f900 20 run: stack=0x5560dbfa0a80 is io blocked 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.510+0000 7fc263fff640 20 link_request req_data=0x5560dbfa61e0 req_data->id=3, curl_handle=0x5560dbfa5a90 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 20 cr:s=0x5560dbf94e40:op=0x5560dbf981b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 20 cr:s=0x5560dbf94e40:op=0x5560dbf981b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 20 cr:s=0x5560dbf94e40:op=0x5560dbf981b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 20 cr:s=0x5560dbf94e40:op=0x5560dbf981b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.522 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 20 cr:s=0x5560dbf94e40:op=0x5560dbf97b90:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 15 stack 0x5560dbf94e40 end 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 20 run: stack=0x5560dbf94e40 is done 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf909a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 20 collect(): s=0x5560dbe56840 stack=0x5560dbf94e40 is complete 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 10 trimming bilogs for bucket=:dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) markers=[00000000002.35.1,00000000002.30.1,00000000002.46.1,00000000002.27.1,00000000002.25.1,00000000002.47.1,00000000002.30.1,00000000002.31.1,00000000002.40.1,00000000002.59.1,00000000002.24.1], shards=11 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbfe1ed0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 10 trimming bilog shard 0 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.35.1 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 10 trimming bilog shard 1 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.30.1 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 10 trimming bilog shard 2 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.46.1 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 10 trimming bilog shard 3 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.27.1 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 10 trimming bilog shard 4 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.25.1 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 10 trimming bilog shard 5 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.47.1 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 10 trimming bilog shard 6 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.30.1 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 10 trimming bilog shard 7 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.31.1 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 10 trimming bilog shard 8 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.40.1 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 10 trimming bilog shard 9 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.59.1 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 10 trimming bilog shard 10 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.24.1 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 20 cr:s=0x5560dbf94e40:op=0x5560dbfa9250:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbf9c3d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 20 cr:s=0x5560dbfa82f0:op=0x5560dbfad900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 20 cr:s=0x5560dbfe0de0:op=0x5560dbfe5610:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 20 cr:s=0x5560dbfae3d0:op=0x5560dbfd9c20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 20 cr:s=0x5560dbf9cbd0:op=0x5560dbf97b90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 20 cr:s=0x5560dbfea310:op=0x5560dbfe4920:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbfe2c40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 20 cr:s=0x5560dbfad750:op=0x5560dbfe3ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 20 cr:s=0x5560dbfe7540:op=0x5560dbfdf350:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 20 cr:s=0x5560dbfda630:op=0x5560dbfdfb50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 20 cr:s=0x5560dbf94e40:op=0x5560dbfa9250:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.0 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 20 run: stack=0x5560dbf94e40 is io blocked 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbf9c3d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.1 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 20 run: stack=0x5560dbfe1630 is io blocked 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.511+0000 7fc30894f900 20 cr:s=0x5560dbfa82f0:op=0x5560dbfad900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.2 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 run: stack=0x5560dbfa82f0 is io blocked 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 cr:s=0x5560dbfe0de0:op=0x5560dbfe5610:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.3 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 run: stack=0x5560dbfe0de0 is io blocked 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 cr:s=0x5560dbfae3d0:op=0x5560dbfd9c20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.4 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 run: stack=0x5560dbfae3d0 is io blocked 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 cr:s=0x5560dbf9cbd0:op=0x5560dbf97b90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.5 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 run: stack=0x5560dbf9cbd0 is io blocked 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 cr:s=0x5560dbfea310:op=0x5560dbfe4920:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.6 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 run: stack=0x5560dbfea310 is io blocked 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbfe2c40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.7 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 run: stack=0x5560dbfe1440 is io blocked 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 cr:s=0x5560dbfad750:op=0x5560dbfe3ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.8 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 run: stack=0x5560dbfad750 is io blocked 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 cr:s=0x5560dbfe7540:op=0x5560dbfdf350:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.9 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 run: stack=0x5560dbfe7540 is io blocked 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 cr:s=0x5560dbfda630:op=0x5560dbfdfb50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.10 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 run: stack=0x5560dbfda630 is io blocked 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 cr:s=0x5560dbfa0a80:op=0x5560dbfa0c10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 cr:s=0x5560dbfa0a80:op=0x5560dbfa0c10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 cr:s=0x5560dbfa0a80:op=0x5560dbfa0c10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 cr:s=0x5560dbfa0a80:op=0x5560dbfa0c10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 cr:s=0x5560dbfa0a80:op=0x5560dbfa0460:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 15 stack 0x5560dbfa0a80 end 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 run: stack=0x5560dbfa0a80 is done 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 collect(): s=0x5560dbea7100 stack=0x5560dbfa0a80 is complete 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 10 min_generation is 1 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.512+0000 7fc30894f900 20 cr:s=0x5560dbf9fe00:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbff98e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.0 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.1 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.2 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.3 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.4 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.5 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.6 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.7 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.8 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.9 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.10 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 run: stack=0x5560dbea7100 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 cr:s=0x5560dbf9fe00:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 cr:s=0x5560dbfa0a80:op=0x5560dbfa03e0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 cr:s=0x5560dbfea4c0:op=0x5560dbfa4630:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbffa530:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 cr:s=0x5560dbfa2db0:op=0x5560dbfa0bc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 cr:s=0x5560dbfa2130:op=0x5560dbfa11b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 cr:s=0x5560dbffad10:op=0x5560dbffbfd0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 cr:s=0x5560dbffbca0:op=0x5560dbffc5c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 cr:s=0x5560dbffb7c0:op=0x5560dbffcbb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 cr:s=0x5560dbffdd00:op=0x5560dbffd310:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 cr:s=0x5560dbffe9a0:op=0x5560dbffdfb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 cr:s=0x5560dbfff640:op=0x5560dbffec50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 cr:s=0x5560dbf9fe00:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 cr:s=0x5560dbfa0a80:op=0x5560dbfa03e0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 run: stack=0x5560dbfa0a80 is io blocked 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 cr:s=0x5560dbfea4c0:op=0x5560dbfa4630:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 run: stack=0x5560dbfea4c0 is io blocked 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbffa530:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 run: stack=0x5560dbfa25b0 is io blocked 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 cr:s=0x5560dbfa2db0:op=0x5560dbfa0bc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 run: stack=0x5560dbfa2db0 is io blocked 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 cr:s=0x5560dbfa2130:op=0x5560dbfa11b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 run: stack=0x5560dbfa2130 is io blocked 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.513+0000 7fc30894f900 20 cr:s=0x5560dbffad10:op=0x5560dbffbfd0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 run: stack=0x5560dbffad10 is io blocked 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 cr:s=0x5560dbffbca0:op=0x5560dbffc5c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 run: stack=0x5560dbffbca0 is io blocked 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 cr:s=0x5560dbffb7c0:op=0x5560dbffcbb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 run: stack=0x5560dbffb7c0 is io blocked 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 cr:s=0x5560dbffdd00:op=0x5560dbffd310:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 run: stack=0x5560dbffdd00 is io blocked 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 cr:s=0x5560dbffe9a0:op=0x5560dbffdfb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 run: stack=0x5560dbffe9a0 is io blocked 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 cr:s=0x5560dbfff640:op=0x5560dbffec50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 run: stack=0x5560dbfff640 is io blocked 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 cr:s=0x5560dbf9fe00:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 cr:s=0x5560dbfae3d0:op=0x5560dbfd9c20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 cr:s=0x5560dbfae3d0:op=0x5560dbfd9c20:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 15 stack 0x5560dbfae3d0 end 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 stack->operate() returned ret=-61 2026-03-20T11:47:02.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 run: stack=0x5560dbfae3d0 is done 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 cr:s=0x5560dbf9cbd0:op=0x5560dbf97b90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 cr:s=0x5560dbf9cbd0:op=0x5560dbf97b90:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 15 stack 0x5560dbf9cbd0 end 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 stack->operate() returned ret=-61 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 run: stack=0x5560dbf9cbd0 is done 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 cr:s=0x5560dbf9fe00:op=0x5560dbf9f700:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 15 stack 0x5560dbf9fe00 end 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 run: stack=0x5560dbf9fe00 is done 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbfe1ed0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 collect(): s=0x5560dbf8e630 stack=0x5560dbf9fe00 is complete 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 10 min_generation is 1 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbfe1ed0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf9f700:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.0 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.1 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.2 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.3 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.4 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.5 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.514+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.6 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.7 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.8 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.9 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.10 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dbf9cbd0:op=0x5560dbf8f500:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dbf9fe00:op=0x5560dbf97b90:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dbfae3d0:op=0x5560dbfd9d90:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dbfed9d0:op=0x5560dbfecfe0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dbfedca0:op=0x5560dc0022c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dc003810:op=0x5560dc002e20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dc0044b0:op=0x5560dc003ac0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dc005150:op=0x5560dc004760:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dc005df0:op=0x5560dc005400:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dc006a90:op=0x5560dc0060a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dc007730:op=0x5560dc006d40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dbf9cbd0:op=0x5560dbf8f500:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 run: stack=0x5560dbf9cbd0 is io blocked 2026-03-20T11:47:02.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dbf9fe00:op=0x5560dbf97b90:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 run: stack=0x5560dbf9fe00 is io blocked 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dbfae3d0:op=0x5560dbfd9d90:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 run: stack=0x5560dbfae3d0 is io blocked 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dbfed9d0:op=0x5560dbfecfe0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 run: stack=0x5560dbfed9d0 is io blocked 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dbfedca0:op=0x5560dc0022c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 run: stack=0x5560dbfedca0 is io blocked 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dc003810:op=0x5560dc002e20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 run: stack=0x5560dc003810 is io blocked 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dc0044b0:op=0x5560dc003ac0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 run: stack=0x5560dc0044b0 is io blocked 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dc005150:op=0x5560dc004760:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 run: stack=0x5560dc005150 is io blocked 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dc005df0:op=0x5560dc005400:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 run: stack=0x5560dc005df0 is io blocked 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dc006a90:op=0x5560dc0060a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 run: stack=0x5560dc006a90 is io blocked 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dc007730:op=0x5560dc006d40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 run: stack=0x5560dc007730 is io blocked 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf970c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 15 stack 0x5560dbf950e0 end 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is done 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 collect(): s=0x5560dbf8ebb0 stack=0x5560dbf950e0 is complete 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 15 stack 0x5560dbf8ebb0 end 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is done 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 4 starting trim on bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbf9c3d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbf9c3d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 15 stack 0x5560dbfe1630 end 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 stack->operate() returned ret=-61 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 run: stack=0x5560dbfe1630 is done 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dbfe7540:op=0x5560dbfdf350:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dbfe7540:op=0x5560dbfdf350:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 15 stack 0x5560dbfe7540 end 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 stack->operate() returned ret=-61 2026-03-20T11:47:02.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 run: stack=0x5560dbfe7540 is done 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 enqueued request req=0x5560dbfe7350 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.515+0000 7fc30894f900 20 req: 0x5560dbfe7350 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.516+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is io blocked 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.516+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbfe1ed0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.516+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbfe1ed0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.516+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.516+0000 7fc2c0ff9640 20 dequeued request req=0x5560dbfe7350 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.516+0000 7fc2c0ff9640 20 RGWWQ: empty 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc2c0ff9640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 : miss 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc2c0ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 cr:s=0x5560dbf94e40:op=0x5560dbfa9250:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 cr:s=0x5560dbf94e40:op=0x5560dbfa9250:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 15 stack 0x5560dbf94e40 end 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 stack->operate() returned ret=-61 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 run: stack=0x5560dbf94e40 is done 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbfe1ed0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 cr:s=0x5560dbfad750:op=0x5560dbfe3ae0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 cr:s=0x5560dbfad750:op=0x5560dbfe3ae0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 15 stack 0x5560dbfad750 end 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 stack->operate() returned ret=-61 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 run: stack=0x5560dbfad750 is done 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 cr:s=0x5560dbfa2130:op=0x5560dbfa11b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 cr:s=0x5560dbfe0de0:op=0x5560dbfe5610:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 cr:s=0x5560dbfe0de0:op=0x5560dbfe5610:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 15 stack 0x5560dbfe0de0 end 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 stack->operate() returned ret=-61 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 run: stack=0x5560dbfe0de0 is done 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 cr:s=0x5560dbfea310:op=0x5560dbfe4920:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 cr:s=0x5560dbfea310:op=0x5560dbfe4920:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 15 stack 0x5560dbfea310 end 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 stack->operate() returned ret=-61 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 run: stack=0x5560dbfea310 is done 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 cr:s=0x5560dbfea4c0:op=0x5560dbfa4630:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 cr:s=0x5560dbfa82f0:op=0x5560dbfad900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 cr:s=0x5560dbfa82f0:op=0x5560dbfad900:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 15 stack 0x5560dbfa82f0 end 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 stack->operate() returned ret=-61 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 run: stack=0x5560dbfa82f0 is done 2026-03-20T11:47:02.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbfe2c40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbfe2c40:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 15 stack 0x5560dbfe1440 end 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 stack->operate() returned ret=-61 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 run: stack=0x5560dbfe1440 is done 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 cr:s=0x5560dbfed9d0:op=0x5560dbfecfe0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbfe1ed0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 cr:s=0x5560dbfa2130:op=0x5560dbfa11b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 cr:s=0x5560dbfea4c0:op=0x5560dbfa4630:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 cr:s=0x5560dbfed9d0:op=0x5560dbfecfe0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.517+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbfe1ed0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc30894f900 20 cr:s=0x5560dbfa2130:op=0x5560dbfa11b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc30894f900 20 cr:s=0x5560dbfea4c0:op=0x5560dbfa4630:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc30894f900 20 cr:s=0x5560dbfed9d0:op=0x5560dbfecfe0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbfe1ed0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc30894f900 20 cr:s=0x5560dbfa2130:op=0x5560dbfa11b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc30894f900 15 stack 0x5560dbfa2130 end 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc30894f900 20 run: stack=0x5560dbfa2130 is done 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc30894f900 20 cr:s=0x5560dbfea4c0:op=0x5560dbfa4630:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc30894f900 15 stack 0x5560dbfea4c0 end 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc30894f900 20 run: stack=0x5560dbfea4c0 is done 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc30894f900 20 cr:s=0x5560dbfed9d0:op=0x5560dbfecfe0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc30894f900 15 stack 0x5560dbfed9d0 end 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc30894f900 20 run: stack=0x5560dbfed9d0 is done 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbfe1ed0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbff98e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf9f700:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbfe1ed0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbff98e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc30894f900 20 run: stack=0x5560dbea7100 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc2c0ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc2c0ff9640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 info.flags=0x17 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc2c0ff9640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 to cache LRU end 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc2c0ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc2c0ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc2c0ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc2c0ff9640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-19 : miss 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc2c0ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc2c0ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc2c0ff9640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-19 info.flags=0x0 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc2c0ff9640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-19 to cache LRU end 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc2c0ff9640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-19 : miss 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.518+0000 7fc2c0ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.519+0000 7fc2c0ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.519+0000 7fc2c0ff9640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-19 info.flags=0x0 2026-03-20T11:47:02.529 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.519+0000 7fc2c0ff9640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-19 to cache LRU end 2026-03-20T11:47:02.530 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.522+0000 7f3ed218a640 -1 req 17022207368389142294 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.530 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.525+0000 7f3ef59d1640 -1 req 201624902651196591 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.530 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.531+0000 7f3eeb1bc640 -1 req 15345587898642963671 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.519+0000 7fc2c0ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24): adding source pipe: {s={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.519+0000 7fc2c0ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24): adding dest pipe: {s={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.534 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.519+0000 7fc2c0ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.519+0000 7fc2c0ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.519+0000 7fc2c0ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.519+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.519+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.519+0000 7fc30894f900 20 cr:s=0x5560dbfda630:op=0x5560dbfdfb50:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.519+0000 7fc30894f900 20 cr:s=0x5560dbfda630:op=0x5560dbfdfb50:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.519+0000 7fc30894f900 15 stack 0x5560dbfda630 end 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.519+0000 7fc30894f900 20 stack->operate() returned ret=-61 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.519+0000 7fc30894f900 20 run: stack=0x5560dbfda630 is done 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.519+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.519+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbfe1ed0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.519+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.519+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf909a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.519+0000 7fc30894f900 20 trim: trimmed bucket instance dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.519+0000 7fc30894f900 15 stack 0x5560dbe56840 end 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.519+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is done 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.519+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.519+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.519+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.519+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf970c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.519+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 4 starting trim on bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbfa11b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 7b38d0d873b7fd04b2a490db7771c3c7a96ed0ae5d94eac90acad62f6535f198 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:7b38d0d873b7fd04b2a490db7771c3c7a96ed0ae5d94eac90acad62f6535f198 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 0fb677256e2659a1262b9d963541326a4594a12a9c2148ad7eecfb5b8e105197 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=0fb677256e2659a1262b9d963541326a4594a12a9c2148ad7eecfb5b8e105197 2026-03-20T11:47:02.535 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=4, curl_handle=0x5560dbf99fc0 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 run: stack=0x5560dbfe1440 is io blocked 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc263fff640 20 link_request req_data=0x5560dbf9aa80 req_data->id=4, curl_handle=0x5560dbf99fc0 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbfa11b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 enqueued request req=0x5560dbf96840 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 req: 0x5560dbf96840 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is io blocked 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbffa530:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 cr:s=0x5560dbfedca0:op=0x5560dc0022c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 cr:s=0x5560dbffdd00:op=0x5560dbffd310:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 cr:s=0x5560dbfae3d0:op=0x5560dbfd9d90:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 cr:s=0x5560dc005150:op=0x5560dc004760:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbffa530:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 cr:s=0x5560dbfedca0:op=0x5560dc0022c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 cr:s=0x5560dbffdd00:op=0x5560dbffd310:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 cr:s=0x5560dbfae3d0:op=0x5560dbfd9d90:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 cr:s=0x5560dc005150:op=0x5560dc004760:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbffa530:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 cr:s=0x5560dbfedca0:op=0x5560dc0022c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc2c07f8640 20 dequeued request req=0x5560dbf96840 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc2c07f8640 20 RGWWQ: empty 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc2c07f8640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 : miss 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc2c07f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 cr:s=0x5560dbffdd00:op=0x5560dbffd310:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 cr:s=0x5560dbfae3d0:op=0x5560dbfd9d90:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 cr:s=0x5560dc005150:op=0x5560dc004760:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbffa530:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 15 stack 0x5560dbfa25b0 end 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 run: stack=0x5560dbfa25b0 is done 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 cr:s=0x5560dbfedca0:op=0x5560dc0022c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 15 stack 0x5560dbfedca0 end 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 run: stack=0x5560dbfedca0 is done 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 cr:s=0x5560dbffdd00:op=0x5560dbffd310:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 15 stack 0x5560dbffdd00 end 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 run: stack=0x5560dbffdd00 is done 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 cr:s=0x5560dbfae3d0:op=0x5560dbfd9d90:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 15 stack 0x5560dbfae3d0 end 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 run: stack=0x5560dbfae3d0 is done 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 cr:s=0x5560dc005150:op=0x5560dc004760:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 15 stack 0x5560dc005150 end 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 run: stack=0x5560dc005150 is done 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.520+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbff98e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf9f700:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dc005df0:op=0x5560dc005400:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc2c07f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc2c07f8640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 info.flags=0x17 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc2c07f8640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 to cache LRU end 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc2c07f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc2c07f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc2c07f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc2c07f8640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-4 : miss 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc2c07f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbff98e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 run: stack=0x5560dbea7100 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf9f700:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dc005df0:op=0x5560dc005400:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.536 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbfa0a80:op=0x5560dbfa03e0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbfa2db0:op=0x5560dbfa0bc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbffb7c0:op=0x5560dbffcbb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbfff640:op=0x5560dbffec50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbf9fe00:op=0x5560dbf97b90:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbf9cbd0:op=0x5560dbf8f500:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbffe9a0:op=0x5560dbffdfb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dc003810:op=0x5560dc002e20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dc007730:op=0x5560dc006d40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbffbca0:op=0x5560dbffc5c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbffad10:op=0x5560dbffbfd0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf9f700:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dc005df0:op=0x5560dc005400:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbfa0a80:op=0x5560dbfa03e0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbfa2db0:op=0x5560dbfa0bc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc2c07f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc2c07f8640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-4 info.flags=0x0 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc2c07f8640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-4 to cache LRU end 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc2c07f8640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-4 : miss 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc2c07f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbffb7c0:op=0x5560dbffcbb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbfff640:op=0x5560dbffec50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbf9fe00:op=0x5560dbf97b90:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbf9cbd0:op=0x5560dbf8f500:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbffe9a0:op=0x5560dbffdfb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dc003810:op=0x5560dc002e20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dc007730:op=0x5560dc006d40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbffbca0:op=0x5560dbffc5c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbffad10:op=0x5560dbffbfd0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dc005df0:op=0x5560dc005400:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 15 stack 0x5560dc005df0 end 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 run: stack=0x5560dc005df0 is done 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbfa0a80:op=0x5560dbfa03e0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbfa2db0:op=0x5560dbfa0bc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbffb7c0:op=0x5560dbffcbb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbfff640:op=0x5560dbffec50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbf9fe00:op=0x5560dbf97b90:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dc006a90:op=0x5560dc0060a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbf9cbd0:op=0x5560dbf8f500:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbffe9a0:op=0x5560dbffdfb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dc003810:op=0x5560dc002e20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dc007730:op=0x5560dc006d40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbffbca0:op=0x5560dbffc5c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbffad10:op=0x5560dbffbfd0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf9f700:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbfa0a80:op=0x5560dbfa03e0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 15 stack 0x5560dbfa0a80 end 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 run: stack=0x5560dbfa0a80 is done 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbfa2db0:op=0x5560dbfa0bc0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 15 stack 0x5560dbfa2db0 end 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 run: stack=0x5560dbfa2db0 is done 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbffb7c0:op=0x5560dbffcbb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 15 stack 0x5560dbffb7c0 end 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 run: stack=0x5560dbffb7c0 is done 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbfff640:op=0x5560dbffec50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 15 stack 0x5560dbfff640 end 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 run: stack=0x5560dbfff640 is done 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbf9fe00:op=0x5560dbf97b90:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 15 stack 0x5560dbf9fe00 end 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 run: stack=0x5560dbf9fe00 is done 2026-03-20T11:47:02.537 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dc006a90:op=0x5560dc0060a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbf9cbd0:op=0x5560dbf8f500:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 15 stack 0x5560dbf9cbd0 end 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 run: stack=0x5560dbf9cbd0 is done 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbffe9a0:op=0x5560dbffdfb0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 15 stack 0x5560dbffe9a0 end 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 run: stack=0x5560dbffe9a0 is done 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dc003810:op=0x5560dc002e20:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 15 stack 0x5560dc003810 end 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 run: stack=0x5560dc003810 is done 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dc0044b0:op=0x5560dc003ac0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dc007730:op=0x5560dc006d40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 15 stack 0x5560dc007730 end 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 run: stack=0x5560dc007730 is done 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbffbca0:op=0x5560dbffc5c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 15 stack 0x5560dbffbca0 end 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc2c07f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc2c07f8640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-4 info.flags=0x0 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc2c07f8640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-4 to cache LRU end 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc2c07f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5): adding source pipe: {s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc2c07f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5): adding dest pipe: {s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc2c07f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc2c07f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 run: stack=0x5560dbffbca0 is done 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbffad10:op=0x5560dbffbfd0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 15 stack 0x5560dbffad10 end 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 run: stack=0x5560dbffad10 is done 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbff98e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf9f700:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dc006a90:op=0x5560dc0060a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dc0044b0:op=0x5560dc003ac0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbff98e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf9f700:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc2c07f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 20 cr:s=0x5560dc006a90:op=0x5560dc0060a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.521+0000 7fc30894f900 15 stack 0x5560dc006a90 end 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 run: stack=0x5560dc006a90 is done 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 cr:s=0x5560dc0044b0:op=0x5560dc003ac0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbff98e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf9f700:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbfa11b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 cr:s=0x5560dc0044b0:op=0x5560dc003ac0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 15 stack 0x5560dc0044b0 end 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 run: stack=0x5560dc0044b0 is done 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbff98e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf9f700:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbfa11b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbff98e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf9f700:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbfa11b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbff98e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf9f700:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbfa11b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbff98e0:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa4630:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f700:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.538 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfa8460:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa4630:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 enqueued request req=0x5560dbf98390 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 req: 0x5560dbf98390 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is io blocked 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc2bfff7640 20 dequeued request req=0x5560dbf98390 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc2bfff7640 20 RGWWQ: empty 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfa8460:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 enqueued request req=0x5560dbf9cf10 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 req: 0x5560dbf9cf10 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 run: stack=0x5560dbea7100 is io blocked 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc2bf7f6640 20 dequeued request req=0x5560dbf9cf10 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc2bf7f6640 20 RGWWQ: empty 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = b97f37df61daf08dde2df705f8e217dc156abe6657c5fe81c097a954a760ec5c 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:b97f37df61daf08dde2df705f8e217dc156abe6657c5fe81c097a954a760ec5c 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = b2da75585c7f3a5ce89d58ed5f0b59b06fb59a39fd33de0ae4d153179f0c7b99 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b2da75585c7f3a5ce89d58ed5f0b59b06fb59a39fd33de0ae4d153179f0c7b99 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=5, curl_handle=0x5560dbfa2840 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc30894f900 20 run: stack=0x5560dbfa25b0 is io blocked 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.522+0000 7fc263fff640 20 link_request req_data=0x5560dbfa2f90 req_data->id=5, curl_handle=0x5560dbfa2840 2026-03-20T11:47:02.539 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.524+0000 7fc2bf7f6640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 info.flags=0x11 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.524+0000 7fc2bf7f6640 10 rgw async rados processor: moving b1.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 to cache LRU end 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.524+0000 7fc2bf7f6640 10 rgw async rados processor: distributing notification oid=b1.rgw.control:notify.6 cni=[op: 0, obj: b1.rgw.meta:root:.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21, ofs0, ns] 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.524+0000 7fc2d37fe640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 257698038013 cookie 93874496031264 notifier 4740 bl.length()=567 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.524+0000 7fc2d37fe640 10 rgw watcher librados: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 info.flags=0x11 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.524+0000 7fc2d37fe640 10 rgw watcher librados: moving b1.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 to cache LRU end 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.524+0000 7fc2bfff7640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 info.flags=0x11 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.524+0000 7fc2bfff7640 10 rgw async rados processor: moving b1.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 to cache LRU end 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.524+0000 7fc2bfff7640 10 rgw async rados processor: distributing notification oid=b1.rgw.control:notify.5 cni=[op: 0, obj: b1.rgw.meta:root:.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10, ofs0, ns] 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc2d37fe640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 257698038014 cookie 93874496047264 notifier 4740 bl.length()=565 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc2d37fe640 10 rgw watcher librados: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 info.flags=0x11 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc2d37fe640 10 rgw watcher librados: moving b1.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 to cache LRU end 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc2bf7f6640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-61[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21]): orig_sources= new_sources= 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc2bf7f6640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-61[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21]): potential sources added=[] removed=[] 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc2bf7f6640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-61[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21]): orig_dests= new_dests= 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc2bf7f6640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-61[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21]): potential dests added=[] removed=[] 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfa8460:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfa8460:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfa8460:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfa8460:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfd9c20:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfd9c20:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc30894f900 20 enqueued request req=0x5560dbb14f40 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc30894f900 20 req: 0x5560dbb14f40 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc30894f900 20 run: stack=0x5560dbea7100 is io blocked 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc2beff5640 20 dequeued request req=0x5560dbb14f40 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc2beff5640 20 RGWWQ: empty 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc2bfff7640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-5[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10]): orig_sources= new_sources= 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc2bfff7640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-5[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10]): potential sources added=[] removed=[] 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc2bfff7640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-5[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10]): orig_dests= new_dests= 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc2bfff7640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-5[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10]): potential dests added=[] removed=[] 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa4630:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa4630:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa4630:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa4630:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbffa530:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbffa530:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc30894f900 20 enqueued request req=0x5560dbb0eaf0 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc30894f900 20 req: 0x5560dbb0eaf0 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.525+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is io blocked 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.526+0000 7fc2be7f4640 20 dequeued request req=0x5560dbb0eaf0 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.526+0000 7fc2be7f4640 20 RGWWQ: empty 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.526+0000 7fc2beff5640 10 rgw async rados processor: removing b1.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 from cache 2026-03-20T11:47:02.540 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.526+0000 7fc2beff5640 10 rgw async rados processor: distributing notification oid=b1.rgw.control:notify.6 cni=[op: 1, obj: b1.rgw.meta:root:.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21, ofs0, ns] 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.526+0000 7fc2d37fe640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 257698038015 cookie 93874496031264 notifier 4740 bl.length()=206 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.527+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfd9c20:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.527+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfd9c20:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.527+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfd9c20:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.527+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfd9c20:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.527+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.527+0000 7fc30894f900 20 trim: trimmed bucket instance dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.527+0000 7fc30894f900 15 stack 0x5560dbea7100 end 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.527+0000 7fc2be7f4640 10 rgw async rados processor: removing b1.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 from cache 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.527+0000 7fc2be7f4640 10 rgw async rados processor: distributing notification oid=b1.rgw.control:notify.5 cni=[op: 1, obj: b1.rgw.meta:root:.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10, ofs0, ns] 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.527+0000 7fc30894f900 20 run: stack=0x5560dbea7100 is done 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.527+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.527+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.527+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.527+0000 7fc30894f900 4 starting trim on bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.527+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfd9c20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.527+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfd9c20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.527+0000 7fc30894f900 20 enqueued request req=0x5560dbfa4c20 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.527+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.527+0000 7fc30894f900 20 req: 0x5560dbfa4c20 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.527+0000 7fc30894f900 20 run: stack=0x5560dbea7100 is io blocked 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.527+0000 7fc2bd7f2640 20 dequeued request req=0x5560dbfa4c20 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.527+0000 7fc2bd7f2640 20 RGWWQ: empty 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.527+0000 7fc2bd7f2640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 : miss 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.527+0000 7fc2bd7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.527+0000 7fc2d2ffd640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 257698038016 cookie 93874496047264 notifier 4740 bl.length()=205 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbffa530:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbffa530:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbffa530:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbffa530:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc30894f900 20 trim: trimmed bucket instance dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc30894f900 15 stack 0x5560dbf8e630 end 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bd7f2640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bd7f2640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 info.flags=0x17 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bd7f2640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 to cache LRU end 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bd7f2640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bd7f2640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bd7f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is done 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc30894f900 4 starting trim on bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bd7f2640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-53 : miss 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bd7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc30894f900 20 enqueued request req=0x5560dbf9a4f0 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc30894f900 20 req: 0x5560dbf9a4f0 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is io blocked 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bcff1640 20 dequeued request req=0x5560dbf9a4f0 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bcff1640 20 RGWWQ: empty 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bcff1640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 : miss 2026-03-20T11:47:02.541 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bcff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bd7f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bd7f2640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-53 info.flags=0x0 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bd7f2640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-53 to cache LRU end 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bd7f2640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-53 : miss 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bd7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bcff1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bcff1640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 info.flags=0x17 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bcff1640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 to cache LRU end 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bcff1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bcff1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bcff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bcff1640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-14 : miss 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bcff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bd7f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bd7f2640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-53 info.flags=0x0 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bd7f2640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-53 to cache LRU end 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bd7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13): adding source pipe: {s={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bd7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13): adding dest pipe: {s={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bd7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bd7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bd7f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bcff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bcff1640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-14 info.flags=0x0 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bcff1640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-14 to cache LRU end 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bcff1640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-14 : miss 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc2bcff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.528+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfd9c20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfd9c20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfd9c20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfd9c20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 cr:s=0x5560dbfea310:op=0x5560dbf99660:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 run: stack=0x5560dbea7100 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 cr:s=0x5560dbfea310:op=0x5560dbfe1e80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 cr:s=0x5560dbfea310:op=0x5560dbfe1e80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.542 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 987227133ee90be1cf2e4900294dbb4a5f2f51d59500bc4fe8076456f24ba92a 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:987227133ee90be1cf2e4900294dbb4a5f2f51d59500bc4fe8076456f24ba92a 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 0a68809d0c9d16077ef94698de05068a8a1e8844890c59e60a6b2cf92e2b3732 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=0a68809d0c9d16077ef94698de05068a8a1e8844890c59e60a6b2cf92e2b3732 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc2bcff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc2bcff1640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-14 info.flags=0x0 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc2bcff1640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-14 to cache LRU end 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc2bcff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19): adding source pipe: {s={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc2bcff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19): adding dest pipe: {s={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=6, curl_handle=0x5560dbfa5a90 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc2bcff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc2bcff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc2bcff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 run: stack=0x5560dbfea310 is io blocked 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc263fff640 20 link_request req_data=0x7fc2a8005ed0 req_data->id=6, curl_handle=0x5560dbfa5a90 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 cr:s=0x5560dbfe0de0:op=0x5560dc000a40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 cr:s=0x5560dbfe0de0:op=0x5560dc001150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 cr:s=0x5560dbfe0de0:op=0x5560dc001150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.543 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.544 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.544 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.544 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.544 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.544 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.544 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.544 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.544 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.544 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.544 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.544 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 6e5f8a3dd57206e7ad29d26eded34a0d0aa7748d34cf4ba43d11bc38adcd83f3 2026-03-20T11:47:02.544 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.544 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.544 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.544 INFO:teuthology.orchestra.run.vm07.stderr:6e5f8a3dd57206e7ad29d26eded34a0d0aa7748d34cf4ba43d11bc38adcd83f3 2026-03-20T11:47:02.544 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.544 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.544 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.544 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.544 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 59170a034044d099ff6a32e491928ae9465c9cb5f25e77d113ccfcb60ebe92ab 2026-03-20T11:47:02.544 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=59170a034044d099ff6a32e491928ae9465c9cb5f25e77d113ccfcb60ebe92ab 2026-03-20T11:47:02.544 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.544 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.544 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.544 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=7, curl_handle=0x5560dbf9d060 2026-03-20T11:47:02.544 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc30894f900 20 run: stack=0x5560dbfe0de0 is io blocked 2026-03-20T11:47:02.544 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.529+0000 7fc263fff640 20 link_request req_data=0x7fc2a4005f20 req_data->id=7, curl_handle=0x5560dbf9d060 2026-03-20T11:47:02.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf970c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc30894f900 15 stack 0x5560dbfe1440 end 2026-03-20T11:47:02.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc30894f900 20 run: stack=0x5560dbfe1440 is done 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc30894f900 20 collect(): s=0x5560dbf8ebb0 stack=0x5560dbfe1440 is complete 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc30894f900 15 stack 0x5560dbf8ebb0 end 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is done 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc30894f900 4 starting trim on bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc30894f900 20 enqueued request req=0x5560dbfe0910 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc30894f900 20 req: 0x5560dbfe0910 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is io blocked 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc2bc7f0640 20 dequeued request req=0x5560dbfe0910 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc2bc7f0640 20 RGWWQ: empty 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc2bc7f0640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 : miss 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc2bc7f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc2bc7f0640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc2bc7f0640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 info.flags=0x17 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc2bc7f0640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 to cache LRU end 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc2bc7f0640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc2bc7f0640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc2bc7f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc2bc7f0640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-36 : miss 2026-03-20T11:47:02.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.563+0000 7fc2bc7f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.563 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc2bc7f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.563 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc2bc7f0640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-36 info.flags=0x0 2026-03-20T11:47:02.563 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc2bc7f0640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-36 to cache LRU end 2026-03-20T11:47:02.563 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc2bc7f0640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-36 : miss 2026-03-20T11:47:02.563 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc2bc7f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.563 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc2bc7f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.563 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc2bc7f0640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-36 info.flags=0x0 2026-03-20T11:47:02.563 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc2bc7f0640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-36 to cache LRU end 2026-03-20T11:47:02.563 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc2bc7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41): adding source pipe: {s={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.563 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc2bc7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41): adding dest pipe: {s={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.563 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc2bc7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.563 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc2bc7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.563 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc2bc7f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:47:02.563 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.563 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.563 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.563 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.563 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.563 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf970c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.563 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.563 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.563 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.563 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.563 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.563 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.563 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = e03a9eba0621e5f7d80e8bd9f26a8d132f724ecfc2dd35232c0e9bc74805af4d 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:e03a9eba0621e5f7d80e8bd9f26a8d132f724ecfc2dd35232c0e9bc74805af4d 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = fbab85fac72189dd8ed66938e6f3c96217535286e793f501c396770138c50da6 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=fbab85fac72189dd8ed66938e6f3c96217535286e793f501c396770138c50da6 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=8, curl_handle=0x5560dbf99fc0 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc30894f900 20 run: stack=0x5560dbfe1440 is io blocked 2026-03-20T11:47:02.564 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.564+0000 7fc263fff640 20 link_request req_data=0x5560dbf9aa80 req_data->id=8, curl_handle=0x5560dbf99fc0 2026-03-20T11:47:02.564 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.566+0000 7f3ef61d2640 -1 req 5411850723888051613 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.565 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.565 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f700:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc30894f900 15 stack 0x5560dbfa25b0 end 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc30894f900 20 run: stack=0x5560dbfa25b0 is done 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc30894f900 20 collect(): s=0x5560dbe56840 stack=0x5560dbfa25b0 is complete 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc30894f900 15 stack 0x5560dbe56840 end 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is done 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc30894f900 4 starting trim on bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc30894f900 20 enqueued request req=0x5560dc0028b0 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc30894f900 20 req: 0x5560dc0028b0 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is io blocked 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc2bbfef640 20 dequeued request req=0x5560dc0028b0 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc2bbfef640 20 RGWWQ: empty 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc2bbfef640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 : miss 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc2bbfef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc2bbfef640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc2bbfef640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 info.flags=0x17 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc2bbfef640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 to cache LRU end 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc2bbfef640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc2bbfef640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc2bbfef640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc2bbfef640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-22 : miss 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.566+0000 7fc2bbfef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc2bbfef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc2bbfef640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-22 info.flags=0x0 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc2bbfef640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-22 to cache LRU end 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc2bbfef640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-22 : miss 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc2bbfef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc2bbfef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc2bbfef640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-22 info.flags=0x0 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc2bbfef640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-22 to cache LRU end 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc2bbfef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27): adding source pipe: {s={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc2bbfef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27): adding dest pipe: {s={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc2bbfef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc2bbfef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc2bbfef640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f700:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = b6f20b701f76a9d88492c6e2149121a46be075252ef0c9034004f3f8d663a04d 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:b6f20b701f76a9d88492c6e2149121a46be075252ef0c9034004f3f8d663a04d 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 6c37231f47d8fbd8447113201b3cc9511cefc3badfc34671fe022c353cf17826 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=6c37231f47d8fbd8447113201b3cc9511cefc3badfc34671fe022c353cf17826 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=9, curl_handle=0x5560dbfa2840 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc30894f900 20 run: stack=0x5560dbfa25b0 is io blocked 2026-03-20T11:47:02.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.567+0000 7fc263fff640 20 link_request req_data=0x5560dbfa2f90 req_data->id=9, curl_handle=0x5560dbfa2840 2026-03-20T11:47:02.567 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.569+0000 7f3dff7e5640 -1 req 947770853428608856 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.569 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 cr:s=0x5560dbfea310:op=0x5560dbfe1e80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.569 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 cr:s=0x5560dbfea310:op=0x5560dbfe1e80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.569 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 cr:s=0x5560dbfea310:op=0x5560dbfe1e80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.569 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 cr:s=0x5560dbfea310:op=0x5560dbfe1e80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.569 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 cr:s=0x5560dbfea310:op=0x5560dbf99660:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 15 stack 0x5560dbfea310 end 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 run: stack=0x5560dbfea310 is done 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 collect(): s=0x5560dbea7100 stack=0x5560dbfea310 is complete 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 10 trimming bilogs for bucket=:dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) markers=[00000000002.33.1,00000000002.61.1,00000000002.33.1,00000000002.31.1,00000000002.38.1,00000000002.35.1,00000000002.63.1,00000000002.38.1,00000000002.34.1,00000000002.52.1,00000000002.51.1], shards=11 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfa4630:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 10 trimming bilog shard 0 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.33.1 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 10 trimming bilog shard 1 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.61.1 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 10 trimming bilog shard 2 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.33.1 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 10 trimming bilog shard 3 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.31.1 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 10 trimming bilog shard 4 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.38.1 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 10 trimming bilog shard 5 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.35.1 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 10 trimming bilog shard 6 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.63.1 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 10 trimming bilog shard 7 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.38.1 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 10 trimming bilog shard 8 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.34.1 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 10 trimming bilog shard 9 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.52.1 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 10 trimming bilog shard 10 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.51.1 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 run: stack=0x5560dbea7100 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 cr:s=0x5560dbfea310:op=0x5560dbfa9250:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 cr:s=0x5560dbf94e40:op=0x5560dbff95d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 cr:s=0x5560dbfe7540:op=0x5560dbfaaa60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbfa1130:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf9c4b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 cr:s=0x5560dbfa1930:op=0x5560dbfa03e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 cr:s=0x5560dbf9ccb0:op=0x5560dbfae010:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 cr:s=0x5560dbe56650:op=0x5560dbfad500:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 cr:s=0x5560dbfdc8e0:op=0x5560dbfa9f60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfe48a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 cr:s=0x5560dbfa0be0:op=0x5560dbfd9c20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 cr:s=0x5560dbfea310:op=0x5560dbfa9250:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.0 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 run: stack=0x5560dbfea310 is io blocked 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 cr:s=0x5560dbf94e40:op=0x5560dbff95d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.1 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 run: stack=0x5560dbf94e40 is io blocked 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 cr:s=0x5560dbfe7540:op=0x5560dbfaaa60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.2 2026-03-20T11:47:02.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 run: stack=0x5560dbfe7540 is io blocked 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbfa1130:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.3 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 run: stack=0x5560dbfe1630 is io blocked 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf9c4b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.4 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is io blocked 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.571+0000 7fc30894f900 20 cr:s=0x5560dbfa1930:op=0x5560dbfa03e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.5 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 run: stack=0x5560dbfa1930 is io blocked 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 cr:s=0x5560dbf9ccb0:op=0x5560dbfae010:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.6 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 run: stack=0x5560dbf9ccb0 is io blocked 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 cr:s=0x5560dbe56650:op=0x5560dbfad500:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.7 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 run: stack=0x5560dbe56650 is io blocked 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 cr:s=0x5560dbfdc8e0:op=0x5560dbfa9f60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.8 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 run: stack=0x5560dbfdc8e0 is io blocked 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfe48a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.9 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 run: stack=0x5560dbfdd1a0 is io blocked 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 cr:s=0x5560dbfa0be0:op=0x5560dbfd9c20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.10 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 run: stack=0x5560dbfa0be0 is io blocked 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 cr:s=0x5560dbfe0de0:op=0x5560dc001150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 cr:s=0x5560dbfe0de0:op=0x5560dc001150:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 cr:s=0x5560dbfe0de0:op=0x5560dc000a40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 15 stack 0x5560dbfe0de0 end 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 run: stack=0x5560dbfe0de0 is done 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 collect(): s=0x5560dbf8e630 stack=0x5560dbfe0de0 is complete 2026-03-20T11:47:02.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 15 stack 0x5560dbf8e630 end 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is done 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 4 starting trim on bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 enqueued request req=0x5560dbf9a4f0 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 req: 0x5560dbf9a4f0 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is io blocked 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc2bb7ee640 20 dequeued request req=0x5560dbf9a4f0 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc2bb7ee640 20 RGWWQ: empty 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc2bb7ee640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 : miss 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.572+0000 7fc2bb7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.573+0000 7fc2bb7ee640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.573+0000 7fc2bb7ee640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 info.flags=0x17 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.573+0000 7fc2bb7ee640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 to cache LRU end 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.573+0000 7fc2bb7ee640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.573+0000 7fc2bb7ee640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.573+0000 7fc2bb7ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.573+0000 7fc2bb7ee640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-25 : miss 2026-03-20T11:47:02.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.573+0000 7fc2bb7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 cr:s=0x5560dbfa1930:op=0x5560dbfa03e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc2bb7ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc2bb7ee640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-25 info.flags=0x0 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc2bb7ee640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-25 to cache LRU end 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc2bb7ee640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-25 : miss 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc2bb7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 cr:s=0x5560dbfa1930:op=0x5560dbfa03e0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 15 stack 0x5560dbfa1930 end 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 stack->operate() returned ret=-61 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 run: stack=0x5560dbfa1930 is done 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfa4630:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 run: stack=0x5560dbea7100 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 cr:s=0x5560dbe56650:op=0x5560dbfad500:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 cr:s=0x5560dbe56650:op=0x5560dbfad500:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 15 stack 0x5560dbe56650 end 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 stack->operate() returned ret=-61 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 run: stack=0x5560dbe56650 is done 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 cr:s=0x5560dbfea310:op=0x5560dbfa9250:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 cr:s=0x5560dbfea310:op=0x5560dbfa9250:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 15 stack 0x5560dbfea310 end 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 stack->operate() returned ret=-61 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 run: stack=0x5560dbfea310 is done 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfa4630:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfa4630:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc2bb7ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc2bb7ee640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-25 info.flags=0x0 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc2bb7ee640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-25 to cache LRU end 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc2bb7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30): adding source pipe: {s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc2bb7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30): adding dest pipe: {s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc2bb7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc2bb7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 run: stack=0x5560dbea7100 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfe48a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfe48a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 15 stack 0x5560dbfdd1a0 end 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 stack->operate() returned ret=-61 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 run: stack=0x5560dbfdd1a0 is done 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfa4630:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 run: stack=0x5560dbea7100 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc2bb7ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.575 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dc001150:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dc000a40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dc000a40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 69750e700565a2b657b7217ab590ed9c83f3f4960a288b1e526fcae81b8df645 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:69750e700565a2b657b7217ab590ed9c83f3f4960a288b1e526fcae81b8df645 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 8be10274e28ac439a21685b33829cdb6bab26e1430ca184f7c5fd1e1b5516914 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=8be10274e28ac439a21685b33829cdb6bab26e1430ca184f7c5fd1e1b5516914 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=10, curl_handle=0x5560dbf9d060 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 run: stack=0x5560dbfdd1a0 is io blocked 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbfa1130:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbfa1130:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 15 stack 0x5560dbfe1630 end 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc263fff640 20 link_request req_data=0x7fc2a4005f20 req_data->id=10, curl_handle=0x5560dbf9d060 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 stack->operate() returned ret=-61 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 run: stack=0x5560dbfe1630 is done 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 cr:s=0x5560dbfe7540:op=0x5560dbfaaa60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.574+0000 7fc30894f900 20 cr:s=0x5560dbfe7540:op=0x5560dbfaaa60:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 15 stack 0x5560dbfe7540 end 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 stack->operate() returned ret=-61 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 run: stack=0x5560dbfe7540 is done 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 cr:s=0x5560dbfdc8e0:op=0x5560dbfa9f60:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 cr:s=0x5560dbfdc8e0:op=0x5560dbfa9f60:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 15 stack 0x5560dbfdc8e0 end 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 stack->operate() returned ret=-61 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 run: stack=0x5560dbfdc8e0 is done 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 cr:s=0x5560dbfa0be0:op=0x5560dbfd9c20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.576 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 cr:s=0x5560dbfa0be0:op=0x5560dbfd9c20:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 15 stack 0x5560dbfa0be0 end 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 stack->operate() returned ret=-61 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 run: stack=0x5560dbfa0be0 is done 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfa4630:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 cr:s=0x5560dbf94e40:op=0x5560dbff95d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 cr:s=0x5560dbf94e40:op=0x5560dbff95d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 15 stack 0x5560dbf94e40 end 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 stack->operate() returned ret=-61 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 run: stack=0x5560dbf94e40 is done 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 cr:s=0x5560dbf9ccb0:op=0x5560dbfae010:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 cr:s=0x5560dbf9ccb0:op=0x5560dbfae010:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 15 stack 0x5560dbf9ccb0 end 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 stack->operate() returned ret=-61 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 run: stack=0x5560dbf9ccb0 is done 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf9c4b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf9c4b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 15 stack 0x5560dbf950e0 end 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 stack->operate() returned ret=-61 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is done 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfa4630:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfa4630:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfa4630:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfa4630:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfa4630:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbfa4630:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 cr:s=0x5560dbea7100:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 trim: trimmed bucket instance dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 15 stack 0x5560dbea7100 end 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 run: stack=0x5560dbea7100 is done 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 4 starting trim on bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 enqueued request req=0x5560dbfdd670 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 req: 0x5560dbfdd670 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is io blocked 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc2bafed640 20 dequeued request req=0x5560dbfdd670 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc2bafed640 20 RGWWQ: empty 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc2bafed640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 : miss 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc2bafed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc2bafed640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc2bafed640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 info.flags=0x17 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc2bafed640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 to cache LRU end 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc2bafed640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc2bafed640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc2bafed640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc2bafed640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc2bafed640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-35 : miss 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.575+0000 7fc2bafed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc2bafed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc2bafed640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-35 info.flags=0x0 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc2bafed640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-35 to cache LRU end 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc2bafed640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-35 : miss 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc2bafed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc2bafed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc2bafed640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-35 info.flags=0x0 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc2bafed640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-35 to cache LRU end 2026-03-20T11:47:02.577 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc2bafed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40): adding source pipe: {s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc2bafed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40): adding dest pipe: {s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc2bafed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc2bafed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc2bafed640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbffa530:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbf99660:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 2a4ef0e193dd21a9715eca3cbece2b98efe8038ced71521925b394494635fb37 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2a4ef0e193dd21a9715eca3cbece2b98efe8038ced71521925b394494635fb37 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 23bffd19a3dd4a69d93512a69f3f283478df8d79c793a4bbb03b4b72b1696ded 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=23bffd19a3dd4a69d93512a69f3f283478df8d79c793a4bbb03b4b72b1696ded 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=11, curl_handle=0x5560dbfa5a90 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.576+0000 7fc30894f900 20 run: stack=0x5560dbfe1630 is io blocked 2026-03-20T11:47:02.578 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.577+0000 7fc263fff640 20 link_request req_data=0x7fc2a8005ed0 req_data->id=11, curl_handle=0x5560dbfa5a90 2026-03-20T11:47:02.579 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.577+0000 7f3e47875640 -1 req 1055804993366749733 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.579 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.578+0000 7f3e18016640 -1 req 8452882542783404335 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.605 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.605 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.606 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf970c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.606 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc30894f900 15 stack 0x5560dbfe1440 end 2026-03-20T11:47:02.606 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc30894f900 20 run: stack=0x5560dbfe1440 is done 2026-03-20T11:47:02.606 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.606 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc30894f900 20 collect(): s=0x5560dbf8ebb0 stack=0x5560dbfe1440 is complete 2026-03-20T11:47:02.606 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.606 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.606 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.606 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.606 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc30894f900 15 stack 0x5560dbf8ebb0 end 2026-03-20T11:47:02.606 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.606 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is done 2026-03-20T11:47:02.606 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.606 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.606 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.606 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.606 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc30894f900 4 starting trim on bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:47:02.606 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfa9250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.606 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfa9250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.606 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc30894f900 20 enqueued request req=0x5560dbfe0910 2026-03-20T11:47:02.606 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.606 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc30894f900 20 req: 0x5560dbfe0910 2026-03-20T11:47:02.606 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is io blocked 2026-03-20T11:47:02.606 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc2ba7ec640 20 dequeued request req=0x5560dbfe0910 2026-03-20T11:47:02.606 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc2ba7ec640 20 RGWWQ: empty 2026-03-20T11:47:02.606 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc2ba7ec640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 : miss 2026-03-20T11:47:02.606 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc2ba7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.607+0000 7fc2ba7ec640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc2ba7ec640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 info.flags=0x17 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc2ba7ec640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 to cache LRU end 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc2ba7ec640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc2ba7ec640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc2ba7ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc2ba7ec640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-26 : miss 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc2ba7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc2ba7ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc2ba7ec640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-26 info.flags=0x0 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc2ba7ec640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-26 to cache LRU end 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc2ba7ec640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-26 : miss 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc2ba7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc2ba7ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc2ba7ec640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-26 info.flags=0x0 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc2ba7ec640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-26 to cache LRU end 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc2ba7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31): adding source pipe: {s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc2ba7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31): adding dest pipe: {s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc2ba7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc2ba7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc2ba7ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfa9250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfa9250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfa9250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfa9250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbfead40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.607 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 451985cddfbc53e2df68dc580d88aef026d1b793c1fc17613d7d9be42d3c340f 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:451985cddfbc53e2df68dc580d88aef026d1b793c1fc17613d7d9be42d3c340f 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 13ece2951a09413af683c51f123463dea943b9ddb332e499ed9f8cff65d9a5a2 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=13ece2951a09413af683c51f123463dea943b9ddb332e499ed9f8cff65d9a5a2 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=12, curl_handle=0x5560dbf99fc0 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc30894f900 20 run: stack=0x5560dbfe1440 is io blocked 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.608+0000 7fc263fff640 20 link_request req_data=0x5560dbf9aa80 req_data->id=12, curl_handle=0x5560dbf99fc0 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f700:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc30894f900 15 stack 0x5560dbfa25b0 end 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc30894f900 20 run: stack=0x5560dbfa25b0 is done 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc30894f900 20 collect(): s=0x5560dbe56840 stack=0x5560dbfa25b0 is complete 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc30894f900 15 stack 0x5560dbe56840 end 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is done 2026-03-20T11:47:02.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc30894f900 4 starting trim on bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:47:02.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbfa9250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbfa9250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc30894f900 20 enqueued request req=0x5560dc0028b0 2026-03-20T11:47:02.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc30894f900 20 req: 0x5560dc0028b0 2026-03-20T11:47:02.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc2b9feb640 20 dequeued request req=0x5560dc0028b0 2026-03-20T11:47:02.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc2b9feb640 20 RGWWQ: empty 2026-03-20T11:47:02.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc2b9feb640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 : miss 2026-03-20T11:47:02.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc2b9feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is io blocked 2026-03-20T11:47:02.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc2b9feb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc2b9feb640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 info.flags=0x17 2026-03-20T11:47:02.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc2b9feb640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 to cache LRU end 2026-03-20T11:47:02.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc2b9feb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc2b9feb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc2b9feb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:47:02.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc2b9feb640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-51 : miss 2026-03-20T11:47:02.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc2b9feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.609 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.610+0000 7f3dfafdc640 -1 req 17600372086334301996 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc2b9feb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc2b9feb640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-51 info.flags=0x0 2026-03-20T11:47:02.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc2b9feb640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-51 to cache LRU end 2026-03-20T11:47:02.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc2b9feb640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-51 : miss 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.610+0000 7fc2b9feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc2b9feb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc2b9feb640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-51 info.flags=0x0 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc2b9feb640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-51 to cache LRU end 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc2b9feb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11): adding source pipe: {s={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc2b9feb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11): adding dest pipe: {s={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc2b9feb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc2b9feb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc2b9feb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbfa9250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbfa9250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbfa9250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbfa9250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 17d7f701e70eaebef48d70d8b727a8c11ce9462316a92f635497a2911bb3d68a 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:17d7f701e70eaebef48d70d8b727a8c11ce9462316a92f635497a2911bb3d68a 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 8bf3cc95d6f04950f1ddcdf285ceeafa0c4baacb4c943cf49dcd8841938ed188 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=8bf3cc95d6f04950f1ddcdf285ceeafa0c4baacb4c943cf49dcd8841938ed188 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.611 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=13, curl_handle=0x5560dbfa2840 2026-03-20T11:47:02.611 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc30894f900 20 run: stack=0x5560dbfa25b0 is io blocked 2026-03-20T11:47:02.611 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.611+0000 7fc263fff640 20 link_request req_data=0x5560dbfa2f90 req_data->id=13, curl_handle=0x5560dbfa2840 2026-03-20T11:47:02.611 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.612+0000 7f3e5b09c640 -1 req 5294243894520014732 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dc000a40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dc000a40:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dc001150:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 15 stack 0x5560dbfdd1a0 end 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 run: stack=0x5560dbfdd1a0 is done 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 collect(): s=0x5560dbf8e630 stack=0x5560dbfdd1a0 is complete 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 15 stack 0x5560dbf8e630 end 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is done 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 4 starting trim on bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa9250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa9250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 enqueued request req=0x5560dbf9a4f0 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 req: 0x5560dbf9a4f0 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc2b97ea640 20 dequeued request req=0x5560dbf9a4f0 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc2b97ea640 20 RGWWQ: empty 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc2b97ea640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 : miss 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc2b97ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.615 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is io blocked 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc2b97ea640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc2b97ea640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 info.flags=0x17 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc2b97ea640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 to cache LRU end 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc2b97ea640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc2b97ea640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc2b97ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc2b97ea640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-55 : miss 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc2b97ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc2b97ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc2b97ea640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-55 info.flags=0x0 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc2b97ea640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-55 to cache LRU end 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc2b97ea640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-55 : miss 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc2b97ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc2b97ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc2b97ea640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-55 info.flags=0x0 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc2b97ea640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-55 to cache LRU end 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc2b97ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding source pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc2b97ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding dest pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc2b97ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc2b97ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc2b97ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa9250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa9250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa9250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa9250:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfa9250:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfad4e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfad4e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.617+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.618+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.618+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.618+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.618+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.618+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.616 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.618+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 949e5c268d4bc26abbfa08c0fd3da50a54f13990a2e5580270678baa13766b59 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.618+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:949e5c268d4bc26abbfa08c0fd3da50a54f13990a2e5580270678baa13766b59 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.618+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.618+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.618+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.618+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.618+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = e22e5a74b90360a44035b3a9a481e8b214bd0c10da3beecdc88cb2fba6e1145c 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.618+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=e22e5a74b90360a44035b3a9a481e8b214bd0c10da3beecdc88cb2fba6e1145c 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.618+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.618+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.618+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.618+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=14, curl_handle=0x5560dbf9d060 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.618+0000 7fc30894f900 20 run: stack=0x5560dbfdd1a0 is io blocked 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.618+0000 7fc263fff640 20 link_request req_data=0x7fc2a4005f20 req_data->id=14, curl_handle=0x5560dbf9d060 2026-03-20T11:47:02.617 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.618+0000 7f3e5d0a0640 -1 req 1484663752310233138 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbf99660:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 15 stack 0x5560dbfe1630 end 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 run: stack=0x5560dbfe1630 is done 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 collect(): s=0x5560dbf950e0 stack=0x5560dbfe1630 is complete 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 15 stack 0x5560dbf950e0 end 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is done 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 4 starting trim on bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbff95d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbff95d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 enqueued request req=0x5560dbfdd670 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.617 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 req: 0x5560dbfdd670 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc2b8fe9640 20 dequeued request req=0x5560dbfdd670 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc2b8fe9640 20 RGWWQ: empty 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc2b8fe9640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 : miss 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc2b8fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is io blocked 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc2b8fe9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc2b8fe9640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 info.flags=0x17 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc2b8fe9640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 to cache LRU end 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc2b8fe9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc2b8fe9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc2b8fe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc2b8fe9640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-28 : miss 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc2b8fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc2b8fe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc2b8fe9640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-28 info.flags=0x0 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc2b8fe9640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-28 to cache LRU end 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc2b8fe9640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-28 : miss 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc2b8fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc2b8fe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc2b8fe9640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-28 info.flags=0x0 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc2b8fe9640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-28 to cache LRU end 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc2b8fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding source pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc2b8fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding dest pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc2b8fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc2b8fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc2b8fe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbff95d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbff95d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbff95d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbff95d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbf99660:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.619+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.620+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.620+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.618 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.620+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.620+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.620+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.620+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = f793cd667a531153d0c7c903c22cd7541759881259a3fa163f308e1c9ddc66a0 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.620+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:f793cd667a531153d0c7c903c22cd7541759881259a3fa163f308e1c9ddc66a0 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.620+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.620+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.620+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.620+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.620+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = fb9754858c01e4af76d0ac866b7acfc75e248580545033c23b30bcf22604eed5 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.620+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=fb9754858c01e4af76d0ac866b7acfc75e248580545033c23b30bcf22604eed5 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.620+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.620+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.620+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.620+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=15, curl_handle=0x5560dbfa5a90 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.620+0000 7fc30894f900 20 run: stack=0x5560dbfe1630 is io blocked 2026-03-20T11:47:02.619 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.620+0000 7fc263fff640 20 link_request req_data=0x7fc2a8005ed0 req_data->id=15, curl_handle=0x5560dbfa5a90 2026-03-20T11:47:02.619 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.620+0000 7f3e35050640 -1 req 8835072412442059295 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.649 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.649 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.649 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbfead40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.649 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc30894f900 15 stack 0x5560dbfe1440 end 2026-03-20T11:47:02.649 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc30894f900 20 run: stack=0x5560dbfe1440 is done 2026-03-20T11:47:02.649 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.649 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc30894f900 20 collect(): s=0x5560dbf8ebb0 stack=0x5560dbfe1440 is complete 2026-03-20T11:47:02.649 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.649 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.649 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.649 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.649 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc30894f900 15 stack 0x5560dbf8ebb0 end 2026-03-20T11:47:02.649 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.649 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is done 2026-03-20T11:47:02.649 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.649 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.649 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.649 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.649 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc30894f900 4 starting trim on bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc30894f900 20 enqueued request req=0x5560dbfe0910 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc30894f900 20 req: 0x5560dbfe0910 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc2b87e8640 20 dequeued request req=0x5560dbfe0910 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc2b87e8640 20 RGWWQ: empty 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc2b87e8640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 : miss 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc2b87e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is io blocked 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc2b87e8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc2b87e8640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 info.flags=0x17 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc2b87e8640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 to cache LRU end 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc2b87e8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc2b87e8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc2b87e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc2b87e8640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-31 : miss 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc2b87e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc2b87e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc2b87e8640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-31 info.flags=0x0 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc2b87e8640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-31 to cache LRU end 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc2b87e8640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-31 : miss 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.651+0000 7fc2b87e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc2b87e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc2b87e8640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-31 info.flags=0x0 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc2b87e8640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-31 to cache LRU end 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc2b87e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding source pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc2b87e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding dest pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.650 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc2b87e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc2b87e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc2b87e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbfead40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = fff4a09a010e8ff67f9f8810d8985d62f42db1b7238ec7323e2462d4ba967cd0 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:fff4a09a010e8ff67f9f8810d8985d62f42db1b7238ec7323e2462d4ba967cd0 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = e00de6d932e03d8e917c3d2a67e6d31dfedb618b4491fe8129c564643846f510 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=e00de6d932e03d8e917c3d2a67e6d31dfedb618b4491fe8129c564643846f510 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=16, curl_handle=0x5560dbf99fc0 2026-03-20T11:47:02.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc30894f900 20 run: stack=0x5560dbfe1440 is io blocked 2026-03-20T11:47:02.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.652+0000 7fc263fff640 20 link_request req_data=0x5560dbf9aa80 req_data->id=16, curl_handle=0x5560dbf99fc0 2026-03-20T11:47:02.652 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.653+0000 7f3e3a05a640 -1 req 13645504224760715367 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 15 stack 0x5560dbfa25b0 end 2026-03-20T11:47:02.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 run: stack=0x5560dbfa25b0 is done 2026-03-20T11:47:02.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 collect(): s=0x5560dbe56840 stack=0x5560dbfa25b0 is complete 2026-03-20T11:47:02.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 15 stack 0x5560dbe56840 end 2026-03-20T11:47:02.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is done 2026-03-20T11:47:02.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 4 starting trim on bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:47:02.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 enqueued request req=0x5560dc0028b0 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 req: 0x5560dc0028b0 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc2b7fe7640 20 dequeued request req=0x5560dc0028b0 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc2b7fe7640 20 RGWWQ: empty 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc2b7fe7640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 : miss 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc2b7fe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is io blocked 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc2b7fe7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc2b7fe7640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 info.flags=0x17 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc2b7fe7640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 to cache LRU end 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc2b7fe7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc2b7fe7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc2b7fe7640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc2b7fe7640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-39 : miss 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc2b7fe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc2b7fe7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc2b7fe7640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-39 info.flags=0x0 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc2b7fe7640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-39 to cache LRU end 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc2b7fe7640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-39 : miss 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc2b7fe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc2b7fe7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc2b7fe7640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-39 info.flags=0x0 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc2b7fe7640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-39 to cache LRU end 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc2b7fe7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding source pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc2b7fe7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding dest pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc2b7fe7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc2b7fe7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc2b7fe7640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 02a5a718badca78d60922f68d4f11f06961fbf8a9d5c934f8a916d0221a2f95d 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.653 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.654 INFO:teuthology.orchestra.run.vm07.stderr:02a5a718badca78d60922f68d4f11f06961fbf8a9d5c934f8a916d0221a2f95d 2026-03-20T11:47:02.654 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.654 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.654 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.654 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.654 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 15a41641885bea99fb81adf4ebc5c4aff1820fb882349520df81f1187c0626d3 2026-03-20T11:47:02.654 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=15a41641885bea99fb81adf4ebc5c4aff1820fb882349520df81f1187c0626d3 2026-03-20T11:47:02.654 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.654 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.654 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.654 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=17, curl_handle=0x5560dbfa2840 2026-03-20T11:47:02.654 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc30894f900 20 run: stack=0x5560dbfa25b0 is io blocked 2026-03-20T11:47:02.654 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.653+0000 7fc263fff640 20 link_request req_data=0x5560dbfa2f90 req_data->id=17, curl_handle=0x5560dbfa2840 2026-03-20T11:47:02.654 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.654+0000 7f3e5c09e640 -1 req 7123825275284302102 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfad4e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfad4e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfa9250:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc30894f900 15 stack 0x5560dbfdd1a0 end 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc30894f900 20 run: stack=0x5560dbfdd1a0 is done 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc30894f900 20 collect(): s=0x5560dbf8e630 stack=0x5560dbfdd1a0 is complete 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc30894f900 15 stack 0x5560dbf8e630 end 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is done 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc30894f900 4 starting trim on bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa03e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa03e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc30894f900 20 enqueued request req=0x5560dbf9a4f0 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc30894f900 20 req: 0x5560dbf9a4f0 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc2b77e6640 20 dequeued request req=0x5560dbf9a4f0 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc2b77e6640 20 RGWWQ: empty 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc2b77e6640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 : miss 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc2b77e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.657 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is io blocked 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc2b77e6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc2b77e6640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 info.flags=0x17 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc2b77e6640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 to cache LRU end 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc2b77e6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc2b77e6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc2b77e6640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc2b77e6640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-8 : miss 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.659+0000 7fc2b77e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc2b77e6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc2b77e6640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-8 info.flags=0x0 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc2b77e6640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-8 to cache LRU end 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc2b77e6640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-8 : miss 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc2b77e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc2b77e6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc2b77e6640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-8 info.flags=0x0 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc2b77e6640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-8 to cache LRU end 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc2b77e6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding source pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc2b77e6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding dest pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc2b77e6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc2b77e6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc2b77e6640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa03e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa03e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa03e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa03e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfa9250:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.658 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfad4e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfad4e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 8dadf366f9420120614a3b76b721e523e3431cf9df9e738562d25ec2c71fe189 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:8dadf366f9420120614a3b76b721e523e3431cf9df9e738562d25ec2c71fe189 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 9a68b9e791b04178ce14b2aa47e9b8362f473e65c063448f7e59dced72f23f79 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=9a68b9e791b04178ce14b2aa47e9b8362f473e65c063448f7e59dced72f23f79 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=18, curl_handle=0x5560dbf9d060 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc30894f900 20 run: stack=0x5560dbfdd1a0 is io blocked 2026-03-20T11:47:02.659 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.660+0000 7fc263fff640 20 link_request req_data=0x7fc2a4005f20 req_data->id=18, curl_handle=0x5560dbf9d060 2026-03-20T11:47:02.660 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.660+0000 7f3dfcfe0640 -1 req 17160404567394332164 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.660 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.660 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.660 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbf99660:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.660 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc30894f900 15 stack 0x5560dbfe1630 end 2026-03-20T11:47:02.660 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc30894f900 20 run: stack=0x5560dbfe1630 is done 2026-03-20T11:47:02.660 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.660 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc30894f900 20 collect(): s=0x5560dbf950e0 stack=0x5560dbfe1630 is complete 2026-03-20T11:47:02.660 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.660 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.660 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.660 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.660 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc30894f900 15 stack 0x5560dbf950e0 end 2026-03-20T11:47:02.660 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.660 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is done 2026-03-20T11:47:02.660 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.660 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.660 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.660 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.660 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc30894f900 4 starting trim on bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:47:02.660 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbff95d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.660 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbff95d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.660 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc30894f900 20 enqueued request req=0x5560dbfdd670 2026-03-20T11:47:02.660 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.660 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc30894f900 20 req: 0x5560dbfdd670 2026-03-20T11:47:02.660 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc2b6fe5640 20 dequeued request req=0x5560dbfdd670 2026-03-20T11:47:02.660 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc2b6fe5640 20 RGWWQ: empty 2026-03-20T11:47:02.660 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc2b6fe5640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 : miss 2026-03-20T11:47:02.660 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc2b6fe5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is io blocked 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc2b6fe5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc2b6fe5640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 info.flags=0x17 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc2b6fe5640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 to cache LRU end 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc2b6fe5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc2b6fe5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc2b6fe5640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc2b6fe5640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-16 : miss 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc2b6fe5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc2b6fe5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc2b6fe5640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-16 info.flags=0x0 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc2b6fe5640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-16 to cache LRU end 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc2b6fe5640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-16 : miss 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.661+0000 7fc2b6fe5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc2b6fe5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc2b6fe5640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-16 info.flags=0x0 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc2b6fe5640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-16 to cache LRU end 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc2b6fe5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding source pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc2b6fe5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding dest pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc2b6fe5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc2b6fe5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc2b6fe5640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbff95d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbff95d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbff95d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbff95d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbf99660:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 0c24183043c38a81bc66028d6dd30e08b34ac7c09eb523777bac595fdfdbfb19 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:0c24183043c38a81bc66028d6dd30e08b34ac7c09eb523777bac595fdfdbfb19 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 1128003fdba98061815be4156eb5084d4b6d84f3f8b8b00e1ff2c7599f45d1f8 2026-03-20T11:47:02.661 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=1128003fdba98061815be4156eb5084d4b6d84f3f8b8b00e1ff2c7599f45d1f8 2026-03-20T11:47:02.662 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.662 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.662 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.662 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=19, curl_handle=0x5560dbfa5a90 2026-03-20T11:47:02.662 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc30894f900 20 run: stack=0x5560dbfe1630 is io blocked 2026-03-20T11:47:02.662 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.662+0000 7fc263fff640 20 link_request req_data=0x7fc2a8005ed0 req_data->id=19, curl_handle=0x5560dbfa5a90 2026-03-20T11:47:02.662 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.663+0000 7f3e648af640 -1 req 9052293519954122704 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.691 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.691 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.691 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbfead40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.691 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc30894f900 15 stack 0x5560dbfe1440 end 2026-03-20T11:47:02.691 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc30894f900 20 run: stack=0x5560dbfe1440 is done 2026-03-20T11:47:02.691 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.691 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc30894f900 20 collect(): s=0x5560dbf8ebb0 stack=0x5560dbfe1440 is complete 2026-03-20T11:47:02.691 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.691 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.691 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.691 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc30894f900 15 stack 0x5560dbf8ebb0 end 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is done 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc30894f900 4 starting trim on bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc30894f900 20 enqueued request req=0x5560dbfe0910 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc30894f900 20 req: 0x5560dbfe0910 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc2b67e4640 20 dequeued request req=0x5560dbfe0910 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc2b67e4640 20 RGWWQ: empty 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc2b67e4640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 : miss 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc2b67e4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is io blocked 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc2b67e4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc2b67e4640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 info.flags=0x17 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc2b67e4640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 to cache LRU end 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc2b67e4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc2b67e4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc2b67e4640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc2b67e4640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-43 : miss 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.693+0000 7fc2b67e4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc2b67e4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc2b67e4640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-43 info.flags=0x0 2026-03-20T11:47:02.692 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc2b67e4640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-43 to cache LRU end 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc2b67e4640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-43 : miss 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc2b67e4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc2b67e4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc2b67e4640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-43 info.flags=0x0 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc2b67e4640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-43 to cache LRU end 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc2b67e4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding source pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc2b67e4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding dest pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc2b67e4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc2b67e4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc2b67e4640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbfead40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 44ae150c5221f2251cc108aeb4d058a9d68711cb0ce710d8668792132f2dbba7 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.693 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.694 INFO:teuthology.orchestra.run.vm07.stderr:44ae150c5221f2251cc108aeb4d058a9d68711cb0ce710d8668792132f2dbba7 2026-03-20T11:47:02.694 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.694 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.694 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.694 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.694 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 8cc41ee67e212033c923495d28f1ed0c0eed45d86155127b9a642d861ac804d9 2026-03-20T11:47:02.694 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=8cc41ee67e212033c923495d28f1ed0c0eed45d86155127b9a642d861ac804d9 2026-03-20T11:47:02.694 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.694 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.694 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.694 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=20, curl_handle=0x5560dbf99fc0 2026-03-20T11:47:02.694 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc30894f900 20 run: stack=0x5560dbfe1440 is io blocked 2026-03-20T11:47:02.694 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.694+0000 7fc263fff640 20 link_request req_data=0x5560dbf9aa80 req_data->id=20, curl_handle=0x5560dbf99fc0 2026-03-20T11:47:02.694 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.694+0000 7f3ec296b640 -1 req 7182327753213564858 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc30894f900 15 stack 0x5560dbfa25b0 end 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc30894f900 20 run: stack=0x5560dbfa25b0 is done 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc30894f900 20 collect(): s=0x5560dbe56840 stack=0x5560dbfa25b0 is complete 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc30894f900 15 stack 0x5560dbe56840 end 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is done 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc30894f900 4 starting trim on bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc30894f900 20 enqueued request req=0x5560dc0028b0 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc30894f900 20 req: 0x5560dc0028b0 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc2b5fe3640 20 dequeued request req=0x5560dc0028b0 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc2b5fe3640 20 RGWWQ: empty 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc2b5fe3640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 : miss 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc2b5fe3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is io blocked 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc2b5fe3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc2b5fe3640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 info.flags=0x17 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc2b5fe3640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 to cache LRU end 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc2b5fe3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc2b5fe3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc2b5fe3640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc2b5fe3640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-10 : miss 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc2b5fe3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc2b5fe3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc2b5fe3640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-10 info.flags=0x0 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc2b5fe3640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-10 to cache LRU end 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc2b5fe3640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-10 : miss 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.695+0000 7fc2b5fe3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc2b5fe3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc2b5fe3640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-10 info.flags=0x0 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc2b5fe3640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-10 to cache LRU end 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc2b5fe3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding source pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc2b5fe3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding dest pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc2b5fe3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc2b5fe3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc2b5fe3640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f6b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = cf919382afd5d73dd41e4be5713eb64fb653b8d96a52e845e8d690a20d0c4e54 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:cf919382afd5d73dd41e4be5713eb64fb653b8d96a52e845e8d690a20d0c4e54 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 725613eb926362b57762085e4a5cf359f735e0535bf1d048d05007c3fd0e4be6 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=725613eb926362b57762085e4a5cf359f735e0535bf1d048d05007c3fd0e4be6 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=21, curl_handle=0x5560dbfa2840 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc30894f900 20 run: stack=0x5560dbfa25b0 is io blocked 2026-03-20T11:47:02.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.696+0000 7fc263fff640 20 link_request req_data=0x5560dbfa2f90 req_data->id=21, curl_handle=0x5560dbfa2840 2026-03-20T11:47:02.696 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.697+0000 7f3eaf144640 -1 req 7695249743079733657 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfad4e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfad4e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfa9250:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc30894f900 15 stack 0x5560dbfdd1a0 end 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc30894f900 20 run: stack=0x5560dbfdd1a0 is done 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc30894f900 20 collect(): s=0x5560dbf8e630 stack=0x5560dbfdd1a0 is complete 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc30894f900 15 stack 0x5560dbf8e630 end 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is done 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc30894f900 4 starting trim on bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa03e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa03e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc30894f900 20 enqueued request req=0x5560dbf9a4f0 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc30894f900 20 req: 0x5560dbf9a4f0 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc2b57e2640 20 dequeued request req=0x5560dbf9a4f0 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc2b57e2640 20 RGWWQ: empty 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc2b57e2640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 : miss 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc2b57e2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is io blocked 2026-03-20T11:47:02.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc2b57e2640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:47:02.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc2b57e2640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 info.flags=0x17 2026-03-20T11:47:02.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc2b57e2640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 to cache LRU end 2026-03-20T11:47:02.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc2b57e2640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc2b57e2640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc2b57e2640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:47:02.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc2b57e2640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-3 : miss 2026-03-20T11:47:02.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc2b57e2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc2b57e2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc2b57e2640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-3 info.flags=0x0 2026-03-20T11:47:02.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc2b57e2640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-3 to cache LRU end 2026-03-20T11:47:02.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc2b57e2640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-3 : miss 2026-03-20T11:47:02.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc2b57e2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc2b57e2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.701+0000 7fc2b57e2640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-3 info.flags=0x0 2026-03-20T11:47:02.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc2b57e2640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-3 to cache LRU end 2026-03-20T11:47:02.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc2b57e2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding source pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc2b57e2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding dest pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc2b57e2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc2b57e2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc2b57e2640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa03e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa03e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa03e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa03e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfa9250:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfad4e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfad4e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = b69879fcfc1917fd259fb4ef7079561607a960fa7074bbf67812dbab7221f208 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:b69879fcfc1917fd259fb4ef7079561607a960fa7074bbf67812dbab7221f208 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = ec9c7350a22e2f81de4d043401a0f48beb7870eaa5d443844f2cce548f7ad72a 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=ec9c7350a22e2f81de4d043401a0f48beb7870eaa5d443844f2cce548f7ad72a 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.702 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=22, curl_handle=0x5560dbf9d060 2026-03-20T11:47:02.702 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc30894f900 20 run: stack=0x5560dbfdd1a0 is io blocked 2026-03-20T11:47:02.702 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.702+0000 7fc263fff640 20 link_request req_data=0x7fc2a4005f20 req_data->id=22, curl_handle=0x5560dbf9d060 2026-03-20T11:47:02.702 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.702+0000 7f3ea8136640 -1 req 13112029818032882642 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.702 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.702 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.702 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbf99660:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.702 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc30894f900 15 stack 0x5560dbfe1630 end 2026-03-20T11:47:02.702 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc30894f900 20 run: stack=0x5560dbfe1630 is done 2026-03-20T11:47:02.702 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.702 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc30894f900 20 collect(): s=0x5560dbf950e0 stack=0x5560dbfe1630 is complete 2026-03-20T11:47:02.702 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.702 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.702 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.702 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc30894f900 15 stack 0x5560dbf950e0 end 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is done 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc30894f900 4 starting trim on bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbff95d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbff95d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc30894f900 20 enqueued request req=0x5560dbfdd670 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc30894f900 20 req: 0x5560dbfdd670 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc2b4fe1640 20 dequeued request req=0x5560dbfdd670 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc2b4fe1640 20 RGWWQ: empty 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc2b4fe1640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 : miss 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc2b4fe1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is io blocked 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc2b4fe1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc2b4fe1640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 info.flags=0x17 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc2b4fe1640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 to cache LRU end 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc2b4fe1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc2b4fe1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc2b4fe1640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc2b4fe1640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-9 : miss 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc2b4fe1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc2b4fe1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc2b4fe1640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-9 info.flags=0x0 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc2b4fe1640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-9 to cache LRU end 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc2b4fe1640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-9 : miss 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.703+0000 7fc2b4fe1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc2b4fe1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc2b4fe1640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-9 info.flags=0x0 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc2b4fe1640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-9 to cache LRU end 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc2b4fe1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding source pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc2b4fe1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding dest pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc2b4fe1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc2b4fe1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc2b4fe1640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbff95d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbff95d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbff95d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbff95d0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbff95d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbff95d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.703 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.704 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.704 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.704 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.704 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.704 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.704 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.704 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.704 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = d52915c24d636550a6ef3cfb71c48cc278d6ac85b9b94d132c121bff0748e77d 2026-03-20T11:47:02.704 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.704 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.704 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.704 INFO:teuthology.orchestra.run.vm07.stderr:d52915c24d636550a6ef3cfb71c48cc278d6ac85b9b94d132c121bff0748e77d 2026-03-20T11:47:02.704 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.704 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.704 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.704 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.704 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = d7442b5368345cd19490cb9f4a0787a0589dcaa38117c08af0ffee9831efbae2 2026-03-20T11:47:02.704 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=d7442b5368345cd19490cb9f4a0787a0589dcaa38117c08af0ffee9831efbae2 2026-03-20T11:47:02.704 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.704 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.704 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.704 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=23, curl_handle=0x5560dbfa5a90 2026-03-20T11:47:02.704 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc30894f900 20 run: stack=0x5560dbfe1630 is io blocked 2026-03-20T11:47:02.704 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.704+0000 7fc263fff640 20 link_request req_data=0x7fc2a8005ed0 req_data->id=23, curl_handle=0x5560dbfa5a90 2026-03-20T11:47:02.704 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.704+0000 7f3e2c83f640 -1 req 10283113563909795960 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbfead40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc30894f900 15 stack 0x5560dbfe1440 end 2026-03-20T11:47:02.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc30894f900 20 run: stack=0x5560dbfe1440 is done 2026-03-20T11:47:02.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc30894f900 20 collect(): s=0x5560dbf8ebb0 stack=0x5560dbfe1440 is complete 2026-03-20T11:47:02.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc30894f900 15 stack 0x5560dbf8ebb0 end 2026-03-20T11:47:02.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is done 2026-03-20T11:47:02.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc30894f900 4 starting trim on bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:47:02.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc30894f900 20 enqueued request req=0x5560dbfe0910 2026-03-20T11:47:02.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc30894f900 20 req: 0x5560dbfe0910 2026-03-20T11:47:02.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is io blocked 2026-03-20T11:47:02.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc2bdff3640 20 dequeued request req=0x5560dbfe0910 2026-03-20T11:47:02.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc2bdff3640 20 RGWWQ: empty 2026-03-20T11:47:02.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc2bdff3640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 : miss 2026-03-20T11:47:02.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc2bdff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc2bdff3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc2bdff3640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 info.flags=0x17 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc2bdff3640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 to cache LRU end 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc2bdff3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc2bdff3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc2bdff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc2bdff3640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-38 : miss 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.735+0000 7fc2bdff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc2bdff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc2bdff3640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-38 info.flags=0x0 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc2bdff3640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-38 to cache LRU end 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc2bdff3640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-38 : miss 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc2bdff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc2bdff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc2bdff3640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-38 info.flags=0x0 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc2bdff3640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-38 to cache LRU end 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc2bdff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding source pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc2bdff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding dest pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc2bdff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc2bdff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc2bdff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf99660:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = dfa58f2d372f76d5f69fbb34ac9251ce9003ab9087e40cdbc4e643133b38e0f3 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.735 INFO:teuthology.orchestra.run.vm07.stderr:dfa58f2d372f76d5f69fbb34ac9251ce9003ab9087e40cdbc4e643133b38e0f3 2026-03-20T11:47:02.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 58d4b3447d1c45024c05ae98c8a4d80eea3dc4b36e7634b59301fb71187d43ac 2026-03-20T11:47:02.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=58d4b3447d1c45024c05ae98c8a4d80eea3dc4b36e7634b59301fb71187d43ac 2026-03-20T11:47:02.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=24, curl_handle=0x5560dbf99fc0 2026-03-20T11:47:02.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc30894f900 20 run: stack=0x5560dbfe1440 is io blocked 2026-03-20T11:47:02.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.736+0000 7fc263fff640 20 link_request req_data=0x5560dbf9aa80 req_data->id=24, curl_handle=0x5560dbf99fc0 2026-03-20T11:47:02.736 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.737+0000 7f3ead941640 -1 req 10527863546303061012 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f6b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc30894f900 15 stack 0x5560dbfa25b0 end 2026-03-20T11:47:02.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc30894f900 20 run: stack=0x5560dbfa25b0 is done 2026-03-20T11:47:02.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc30894f900 20 collect(): s=0x5560dbe56840 stack=0x5560dbfa25b0 is complete 2026-03-20T11:47:02.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc30894f900 15 stack 0x5560dbe56840 end 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is done 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc30894f900 4 starting trim on bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc30894f900 20 enqueued request req=0x5560dc0028b0 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc30894f900 20 req: 0x5560dc0028b0 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc2b47e0640 20 dequeued request req=0x5560dc0028b0 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc2b47e0640 20 RGWWQ: empty 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc2b47e0640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 : miss 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc2b47e0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is io blocked 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc2b47e0640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc2b47e0640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 info.flags=0x17 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc2b47e0640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 to cache LRU end 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc2b47e0640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc2b47e0640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc2b47e0640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc2b47e0640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-15 : miss 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc2b47e0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc2b47e0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc2b47e0640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-15 info.flags=0x0 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc2b47e0640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-15 to cache LRU end 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc2b47e0640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-15 : miss 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc2b47e0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.737 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.739+0000 7f3e1380d640 -1 req 15432537696594717981 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc2b47e0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc2b47e0640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-15 info.flags=0x0 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc2b47e0640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-15 to cache LRU end 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.738+0000 7fc2b47e0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding source pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc2b47e0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding dest pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc2b47e0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc2b47e0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc2b47e0640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f6b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = cb08c4ea0ac682f3649630f35ffad7f4c81e3bb45ae9103e20d6cae09563f406 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:cb08c4ea0ac682f3649630f35ffad7f4c81e3bb45ae9103e20d6cae09563f406 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = da61670495d2a93307b6ea8354e568268564e0126bb78c276e310a17fb0bee1e 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=da61670495d2a93307b6ea8354e568268564e0126bb78c276e310a17fb0bee1e 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=25, curl_handle=0x5560dbfa2840 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc30894f900 20 run: stack=0x5560dbfa25b0 is io blocked 2026-03-20T11:47:02.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.739+0000 7fc263fff640 20 link_request req_data=0x5560dbfa2f90 req_data->id=25, curl_handle=0x5560dbfa2840 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfad4e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfad4e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfa9250:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc30894f900 15 stack 0x5560dbfdd1a0 end 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc30894f900 20 run: stack=0x5560dbfdd1a0 is done 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc30894f900 20 collect(): s=0x5560dbf8e630 stack=0x5560dbfdd1a0 is complete 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc30894f900 15 stack 0x5560dbf8e630 end 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is done 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc30894f900 4 starting trim on bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa03e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa03e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc30894f900 20 enqueued request req=0x5560dbf9a4f0 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc30894f900 20 req: 0x5560dbf9a4f0 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is io blocked 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc2b3fdf640 20 dequeued request req=0x5560dbf9a4f0 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc2b3fdf640 20 RGWWQ: empty 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc2b3fdf640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 : miss 2026-03-20T11:47:02.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc2b3fdf640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc2b3fdf640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc2b3fdf640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 info.flags=0x17 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc2b3fdf640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 to cache LRU end 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc2b3fdf640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc2b3fdf640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc2b3fdf640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc2b3fdf640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-13 : miss 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc2b3fdf640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc2b3fdf640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc2b3fdf640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-13 info.flags=0x0 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc2b3fdf640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-13 to cache LRU end 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc2b3fdf640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-13 : miss 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.743+0000 7fc2b3fdf640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc2b3fdf640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc2b3fdf640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-13 info.flags=0x0 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc2b3fdf640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-13 to cache LRU end 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc2b3fdf640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18): adding source pipe: {s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc2b3fdf640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18): adding dest pipe: {s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc2b3fdf640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc2b3fdf640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc2b3fdf640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa03e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa03e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa03e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfa03e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfad4e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfa03e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfa03e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 0909d538088a4e3b46223bb9f5d4df76afda5fe65699c58d55f27162ae6f4017 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:0909d538088a4e3b46223bb9f5d4df76afda5fe65699c58d55f27162ae6f4017 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 2c9fb99bc2a8b26fef67265049a4fe92025f4b569bd9c5afecc589491b885524 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2c9fb99bc2a8b26fef67265049a4fe92025f4b569bd9c5afecc589491b885524 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=26, curl_handle=0x5560dbf9d060 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc30894f900 20 run: stack=0x5560dbfdd1a0 is io blocked 2026-03-20T11:47:02.743 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.744+0000 7fc263fff640 20 link_request req_data=0x7fc2a4005f20 req_data->id=26, curl_handle=0x5560dbf9d060 2026-03-20T11:47:02.743 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.744+0000 7f3ea5130640 -1 req 11450478686798687560 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbff95d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbff95d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc30894f900 15 stack 0x5560dbfe1630 end 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc30894f900 20 run: stack=0x5560dbfe1630 is done 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc30894f900 20 collect(): s=0x5560dbf950e0 stack=0x5560dbfe1630 is complete 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc30894f900 15 stack 0x5560dbf950e0 end 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is done 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc30894f900 4 starting trim on bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc30894f900 20 enqueued request req=0x5560dbfdd670 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc30894f900 20 req: 0x5560dbfdd670 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is io blocked 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc2c37fe640 20 dequeued request req=0x5560dbfdd670 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc2c37fe640 20 RGWWQ: empty 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc2c37fe640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 : miss 2026-03-20T11:47:02.744 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc2c37fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc2c37fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc2c37fe640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 info.flags=0x17 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc2c37fe640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 to cache LRU end 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc2c37fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc2c37fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc2c37fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc2c37fe640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-11 : miss 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc2c37fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc2c37fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc2c37fe640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-11 info.flags=0x0 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc2c37fe640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-11 to cache LRU end 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc2c37fe640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-11 : miss 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc2c37fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc2c37fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc2c37fe640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-11 info.flags=0x0 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc2c37fe640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-11 to cache LRU end 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc2c37fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16): adding source pipe: {s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc2c37fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16): adding dest pipe: {s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc2c37fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc2c37fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.746+0000 7fc2c37fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbff95d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbff95d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.745 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.746 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.746 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.746 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.746 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.746 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.746 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.746 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.746 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.746 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.746 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.746 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = d060cceb88fb1ed380291fc2861be4334ad27308c875fa58e31c192a117330bf 2026-03-20T11:47:02.746 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.746 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.746 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.746 INFO:teuthology.orchestra.run.vm07.stderr:d060cceb88fb1ed380291fc2861be4334ad27308c875fa58e31c192a117330bf 2026-03-20T11:47:02.746 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.746 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.746 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.746 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.746 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 63d6aea4e506eacea03832c5c5c9108d5927780036c3b59fcdb1e89c076b9d57 2026-03-20T11:47:02.746 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=63d6aea4e506eacea03832c5c5c9108d5927780036c3b59fcdb1e89c076b9d57 2026-03-20T11:47:02.746 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.746 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.746 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.746 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=27, curl_handle=0x5560dbfa5a90 2026-03-20T11:47:02.746 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc30894f900 20 run: stack=0x5560dbfe1630 is io blocked 2026-03-20T11:47:02.746 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.747+0000 7fc263fff640 20 link_request req_data=0x7fc2a8005ed0 req_data->id=27, curl_handle=0x5560dbfa5a90 2026-03-20T11:47:02.746 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.747+0000 7f3e620aa640 -1 req 4043907237596257070 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.775 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.775 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.775 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf99660:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.775 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc30894f900 15 stack 0x5560dbfe1440 end 2026-03-20T11:47:02.775 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc30894f900 20 run: stack=0x5560dbfe1440 is done 2026-03-20T11:47:02.775 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.775 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc30894f900 20 collect(): s=0x5560dbf8ebb0 stack=0x5560dbfe1440 is complete 2026-03-20T11:47:02.775 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.775 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.775 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.775 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.775 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc30894f900 15 stack 0x5560dbf8ebb0 end 2026-03-20T11:47:02.775 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.775 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is done 2026-03-20T11:47:02.775 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.775 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.775 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.775 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.775 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc30894f900 4 starting trim on bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:47:02.775 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.775 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.775 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc30894f900 20 enqueued request req=0x5560dbfe0910 2026-03-20T11:47:02.775 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.775 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc30894f900 20 req: 0x5560dbfe0910 2026-03-20T11:47:02.775 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is io blocked 2026-03-20T11:47:02.775 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc2c17fa640 20 dequeued request req=0x5560dbfe0910 2026-03-20T11:47:02.775 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc2c17fa640 20 RGWWQ: empty 2026-03-20T11:47:02.775 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc2c17fa640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 : miss 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc2c17fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc2c17fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc2c17fa640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 info.flags=0x17 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc2c17fa640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 to cache LRU end 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc2c17fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc2c17fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc2c17fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc2c17fa640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-12 : miss 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc2c17fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc2c17fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc2c17fa640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-12 info.flags=0x0 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc2c17fa640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-12 to cache LRU end 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc2c17fa640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-12 : miss 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.777+0000 7fc2c17fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc2c17fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc2c17fa640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-12 info.flags=0x0 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc2c17fa640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-12 to cache LRU end 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc2c17fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17): adding source pipe: {s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc2c17fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17): adding dest pipe: {s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc2c17fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc2c17fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc2c17fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbfa9250:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.776 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 20a9b5ea82ffa48324ba1ee9be15d77f3d592989342a8f3e62a87b0a4494fb89 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:20a9b5ea82ffa48324ba1ee9be15d77f3d592989342a8f3e62a87b0a4494fb89 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = f370a407894478c812db3223fbdf17c7df4cb4dab4339fcd289ebde46dc2fece 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=f370a407894478c812db3223fbdf17c7df4cb4dab4339fcd289ebde46dc2fece 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=28, curl_handle=0x5560dbf99fc0 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc30894f900 20 run: stack=0x5560dbfe1440 is io blocked 2026-03-20T11:47:02.777 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.778+0000 7fc263fff640 20 link_request req_data=0x5560dbf9aa80 req_data->id=28, curl_handle=0x5560dbf99fc0 2026-03-20T11:47:02.777 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.778+0000 7f3e067f3640 -1 req 2605520880943252765 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f6b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc30894f900 15 stack 0x5560dbfa25b0 end 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc30894f900 20 run: stack=0x5560dbfa25b0 is done 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc30894f900 20 collect(): s=0x5560dbe56840 stack=0x5560dbfa25b0 is complete 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc30894f900 15 stack 0x5560dbe56840 end 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is done 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc30894f900 4 starting trim on bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc30894f900 20 enqueued request req=0x5560dc0028b0 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc30894f900 20 req: 0x5560dc0028b0 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is io blocked 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc2c1ffb640 20 dequeued request req=0x5560dc0028b0 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc2c1ffb640 20 RGWWQ: empty 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc2c1ffb640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 : miss 2026-03-20T11:47:02.778 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc2c1ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc2c1ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc2c1ffb640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 info.flags=0x17 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc2c1ffb640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 to cache LRU end 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc2c1ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc2c1ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc2c1ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc2c1ffb640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-40 : miss 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc2c1ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc2c1ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc2c1ffb640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-40 info.flags=0x0 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc2c1ffb640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-40 to cache LRU end 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc2c1ffb640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-40 : miss 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.780+0000 7fc2c1ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc2c1ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc2c1ffb640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-40 info.flags=0x0 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc2c1ffb640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-40 to cache LRU end 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc2c1ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1): adding source pipe: {s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc2c1ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1): adding dest pipe: {s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc2c1ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc2c1ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc2c1ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f6b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.779 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.780 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.780 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.780 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.780 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.780 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.780 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.780 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 1876b06ebac6c53c1a42296085c7a4ad1946e6d624d0decced914ffe829847c4 2026-03-20T11:47:02.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.780 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.780 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.780 INFO:teuthology.orchestra.run.vm07.stderr:1876b06ebac6c53c1a42296085c7a4ad1946e6d624d0decced914ffe829847c4 2026-03-20T11:47:02.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 3cb398aebeb7c3c85af176f7cc8c9f8f2027a61ba657288e5ca75e69d372490d 2026-03-20T11:47:02.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=3cb398aebeb7c3c85af176f7cc8c9f8f2027a61ba657288e5ca75e69d372490d 2026-03-20T11:47:02.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=29, curl_handle=0x5560dbfa2840 2026-03-20T11:47:02.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc30894f900 20 run: stack=0x5560dbfa25b0 is io blocked 2026-03-20T11:47:02.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.781+0000 7fc263fff640 20 link_request req_data=0x5560dbfa2f90 req_data->id=29, curl_handle=0x5560dbfa2840 2026-03-20T11:47:02.780 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.781+0000 7f3e07ff6640 -1 req 9343756958709134117 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfa03e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfa03e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfad4e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 15 stack 0x5560dbfdd1a0 end 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 run: stack=0x5560dbfdd1a0 is done 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 collect(): s=0x5560dbf8e630 stack=0x5560dbfdd1a0 is complete 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 15 stack 0x5560dbf8e630 end 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is done 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 4 starting trim on bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfe3ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfe3ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 enqueued request req=0x5560dbf9a4f0 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 req: 0x5560dbf9a4f0 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is io blocked 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc2c27fc640 20 dequeued request req=0x5560dbf9a4f0 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc2c27fc640 20 RGWWQ: empty 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc2c27fc640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 : miss 2026-03-20T11:47:02.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc2c27fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc2c27fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc2c27fc640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 info.flags=0x17 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc2c27fc640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 to cache LRU end 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc2c27fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc2c27fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc2c27fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc2c27fc640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-18 : miss 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc2c27fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc2c27fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc2c27fc640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-18 info.flags=0x0 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc2c27fc640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-18 to cache LRU end 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc2c27fc640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-18 : miss 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc2c27fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc2c27fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc2c27fc640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-18 info.flags=0x0 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc2c27fc640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-18 to cache LRU end 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc2c27fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23): adding source pipe: {s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc2c27fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23): adding dest pipe: {s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc2c27fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc2c27fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc2c27fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfe3ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfe3ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfe3ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfe3ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfad4e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfa03e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfa03e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.784 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.785 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.785 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.785 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.785 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.785 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.785 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.785 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.785 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 9c95a921af24cb6bdb8d5946f72336569e0855d7e3c2f67797f0e6d25ab47c69 2026-03-20T11:47:02.785 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.785 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.785 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.785 INFO:teuthology.orchestra.run.vm07.stderr:9c95a921af24cb6bdb8d5946f72336569e0855d7e3c2f67797f0e6d25ab47c69 2026-03-20T11:47:02.785 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.785 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.785 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.785 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.785 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 623581526b944707db52d7cadce65f2a930d209f3bab9d3e28e9adfb0729ccac 2026-03-20T11:47:02.785 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=623581526b944707db52d7cadce65f2a930d209f3bab9d3e28e9adfb0729ccac 2026-03-20T11:47:02.785 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.785 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.785 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.785 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=30, curl_handle=0x5560dbf9d060 2026-03-20T11:47:02.785 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc30894f900 20 run: stack=0x5560dbfdd1a0 is io blocked 2026-03-20T11:47:02.785 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.785+0000 7fc263fff640 20 link_request req_data=0x7fc2a4005f20 req_data->id=30, curl_handle=0x5560dbf9d060 2026-03-20T11:47:02.785 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.786+0000 7f3e9310c640 -1 req 7988067122887933368 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbff95d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbff95d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc30894f900 15 stack 0x5560dbfe1630 end 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc30894f900 20 run: stack=0x5560dbfe1630 is done 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc30894f900 20 collect(): s=0x5560dbf950e0 stack=0x5560dbfe1630 is complete 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc30894f900 15 stack 0x5560dbf950e0 end 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is done 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc30894f900 4 starting trim on bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc30894f900 20 enqueued request req=0x5560dbfdd670 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc30894f900 20 req: 0x5560dbfdd670 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is io blocked 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc2c2ffd640 20 dequeued request req=0x5560dbfdd670 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc2c2ffd640 20 RGWWQ: empty 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc2c2ffd640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 : miss 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc2c2ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc2c2ffd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc2c2ffd640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 info.flags=0x17 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc2c2ffd640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 to cache LRU end 2026-03-20T11:47:02.786 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc2c2ffd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc2c2ffd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc2c2ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc2c2ffd640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-50 : miss 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc2c2ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc2c2ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc2c2ffd640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-50 info.flags=0x0 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc2c2ffd640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-50 to cache LRU end 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc2c2ffd640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-50 : miss 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.788+0000 7fc2c2ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc2c2ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc2c2ffd640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-50 info.flags=0x0 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc2c2ffd640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-50 to cache LRU end 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc2c2ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10): adding source pipe: {s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc2c2ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10): adding dest pipe: {s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc2c2ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc2c2ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc2c2ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbff95d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.787 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = d413d7a627d27060540b47ab0b30f737ccb2970c672a92bf6c4f6c8871a6e89d 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr:d413d7a627d27060540b47ab0b30f737ccb2970c672a92bf6c4f6c8871a6e89d 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = dcf8c2ca036de6e006ff1db3430becf113485001ee5ed0eedda9c9118446afb1 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=dcf8c2ca036de6e006ff1db3430becf113485001ee5ed0eedda9c9118446afb1 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=31, curl_handle=0x5560dbfa5a90 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc30894f900 20 run: stack=0x5560dbfe1630 is io blocked 2026-03-20T11:47:02.788 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.789+0000 7fc263fff640 20 link_request req_data=0x7fc2a8005ed0 req_data->id=31, curl_handle=0x5560dbfa5a90 2026-03-20T11:47:02.788 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.790+0000 7f3e9a11a640 -1 req 931104914167021924 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbfa9250:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc30894f900 15 stack 0x5560dbfe1440 end 2026-03-20T11:47:02.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc30894f900 20 run: stack=0x5560dbfe1440 is done 2026-03-20T11:47:02.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc30894f900 20 collect(): s=0x5560dbf8ebb0 stack=0x5560dbfe1440 is complete 2026-03-20T11:47:02.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc30894f900 15 stack 0x5560dbf8ebb0 end 2026-03-20T11:47:02.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is done 2026-03-20T11:47:02.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc30894f900 4 starting trim on bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:47:02.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc30894f900 20 enqueued request req=0x5560dbfe0910 2026-03-20T11:47:02.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc30894f900 20 req: 0x5560dbfe0910 2026-03-20T11:47:02.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is io blocked 2026-03-20T11:47:02.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc2c0ff9640 20 dequeued request req=0x5560dbfe0910 2026-03-20T11:47:02.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc2c0ff9640 20 RGWWQ: empty 2026-03-20T11:47:02.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc2c0ff9640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 : miss 2026-03-20T11:47:02.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc2c0ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc2c0ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc2c0ff9640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 info.flags=0x17 2026-03-20T11:47:02.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc2c0ff9640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 to cache LRU end 2026-03-20T11:47:02.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc2c0ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc2c0ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc2c0ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:47:02.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc2c0ff9640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-24 : miss 2026-03-20T11:47:02.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc2c0ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.818 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.820+0000 7f3ebd961640 -1 req 12561058026440541816 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc2c0ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc2c0ff9640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-24 info.flags=0x0 2026-03-20T11:47:02.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc2c0ff9640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-24 to cache LRU end 2026-03-20T11:47:02.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc2c0ff9640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-24 : miss 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc2c0ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc2c0ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc2c0ff9640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-24 info.flags=0x0 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc2c0ff9640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-24 to cache LRU end 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc2c0ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29): adding source pipe: {s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.819+0000 7fc2c0ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29): adding dest pipe: {s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc2c0ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc2c0ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc2c0ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf987e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf987e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf987e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 771fd963ad7f6532c17481033369ad0ff48a81e003c13d8100b932a5b1f74218 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:771fd963ad7f6532c17481033369ad0ff48a81e003c13d8100b932a5b1f74218 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = f6e89e6a575f56f12c4374c1f66bce9896f1378bd6293ba3e67aa662f7a7d22a 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=f6e89e6a575f56f12c4374c1f66bce9896f1378bd6293ba3e67aa662f7a7d22a 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=32, curl_handle=0x5560dbf99fc0 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc30894f900 20 run: stack=0x5560dbfe1440 is io blocked 2026-03-20T11:47:02.819 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.820+0000 7fc263fff640 20 link_request req_data=0x5560dbf9aa80 req_data->id=32, curl_handle=0x5560dbf99fc0 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f6b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc30894f900 15 stack 0x5560dbfa25b0 end 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc30894f900 20 run: stack=0x5560dbfa25b0 is done 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc30894f900 20 collect(): s=0x5560dbe56840 stack=0x5560dbfa25b0 is complete 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc30894f900 15 stack 0x5560dbe56840 end 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is done 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc30894f900 4 starting trim on bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc30894f900 20 enqueued request req=0x5560dc0028b0 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc30894f900 20 req: 0x5560dc0028b0 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is io blocked 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc2c07f8640 20 dequeued request req=0x5560dc0028b0 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc2c07f8640 20 RGWWQ: empty 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc2c07f8640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 : miss 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc2c07f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc2c07f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc2c07f8640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 info.flags=0x17 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc2c07f8640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 to cache LRU end 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc2c07f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc2c07f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc2c07f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc2c07f8640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-56 : miss 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc2c07f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc2c07f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc2c07f8640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-56 info.flags=0x0 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc2c07f8640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-56 to cache LRU end 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc2c07f8640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-56 : miss 2026-03-20T11:47:02.820 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.822+0000 7fc2c07f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc2c07f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc2c07f8640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-56 info.flags=0x0 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc2c07f8640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-56 to cache LRU end 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc2c07f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16): adding source pipe: {s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc2c07f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16): adding dest pipe: {s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc2c07f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc2c07f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc2c07f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = ce5006aaec97aca15ef0e1fe905bbc0485b16f016a046cdc6e2490a350ed507e 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.821 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.822 INFO:teuthology.orchestra.run.vm07.stderr:ce5006aaec97aca15ef0e1fe905bbc0485b16f016a046cdc6e2490a350ed507e 2026-03-20T11:47:02.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = afec2612b90077b4fe94b8ae3f56318ef4e1c8d2b810253922b26401b10f9d88 2026-03-20T11:47:02.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=afec2612b90077b4fe94b8ae3f56318ef4e1c8d2b810253922b26401b10f9d88 2026-03-20T11:47:02.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=33, curl_handle=0x5560dbfa2840 2026-03-20T11:47:02.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc30894f900 20 run: stack=0x5560dbfa25b0 is io blocked 2026-03-20T11:47:02.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.823+0000 7fc263fff640 20 link_request req_data=0x5560dbfa2f90 req_data->id=33, curl_handle=0x5560dbfa2840 2026-03-20T11:47:02.822 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.823+0000 7f3e6b0bc640 -1 req 13640048764002088466 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfa03e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfa03e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfad4e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc30894f900 15 stack 0x5560dbfdd1a0 end 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc30894f900 20 run: stack=0x5560dbfdd1a0 is done 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc30894f900 20 collect(): s=0x5560dbf8e630 stack=0x5560dbfdd1a0 is complete 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc30894f900 15 stack 0x5560dbf8e630 end 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is done 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc30894f900 4 starting trim on bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfe3ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfe3ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc30894f900 20 enqueued request req=0x5560dbf9a4f0 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc30894f900 20 req: 0x5560dbf9a4f0 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is io blocked 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc2bf7f6640 20 dequeued request req=0x5560dbf9a4f0 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc2bf7f6640 20 RGWWQ: empty 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc2bf7f6640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 : miss 2026-03-20T11:47:02.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc2bf7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc2bf7f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc2bf7f6640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 info.flags=0x17 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc2bf7f6640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 to cache LRU end 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc2bf7f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc2bf7f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc2bf7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc2bf7f6640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-27 : miss 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc2bf7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc2bf7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc2bf7f6640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-27 info.flags=0x0 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc2bf7f6640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-27 to cache LRU end 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc2bf7f6640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-27 : miss 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.827+0000 7fc2bf7f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc2bf7f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc2bf7f6640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-27 info.flags=0x0 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc2bf7f6640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-27 to cache LRU end 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc2bf7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32): adding source pipe: {s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc2bf7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32): adding dest pipe: {s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc2bf7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc2bf7f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc2bf7f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfe3ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfe3ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfe3ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfe3ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfa03e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfad4e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfad4e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.826 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 907baad51beaac1f3ee527f1c558d99bd10f654c249f81dd9dda10c43aec4f9f 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr:907baad51beaac1f3ee527f1c558d99bd10f654c249f81dd9dda10c43aec4f9f 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 33d67e33f62214404af57e53ea0905a5884c1c0be665c3a75d58db21ac347767 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=33d67e33f62214404af57e53ea0905a5884c1c0be665c3a75d58db21ac347767 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=34, curl_handle=0x5560dbf9d060 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc30894f900 20 run: stack=0x5560dbfdd1a0 is io blocked 2026-03-20T11:47:02.827 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.828+0000 7fc263fff640 20 link_request req_data=0x7fc2a4005f20 req_data->id=34, curl_handle=0x5560dbf9d060 2026-03-20T11:47:02.827 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.828+0000 7f3e28837640 -1 req 1351733931678995523 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbff95d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc30894f900 15 stack 0x5560dbfe1630 end 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc30894f900 20 run: stack=0x5560dbfe1630 is done 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc30894f900 20 collect(): s=0x5560dbf950e0 stack=0x5560dbfe1630 is complete 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc30894f900 15 stack 0x5560dbf950e0 end 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is done 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc30894f900 4 starting trim on bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc30894f900 20 enqueued request req=0x5560dbfdd670 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc30894f900 20 req: 0x5560dbfdd670 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is io blocked 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc2bfff7640 20 dequeued request req=0x5560dbfdd670 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc2bfff7640 20 RGWWQ: empty 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc2bfff7640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 : miss 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc2bfff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc2bfff7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc2bfff7640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 info.flags=0x17 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc2bfff7640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 to cache LRU end 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc2bfff7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc2bfff7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc2bfff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc2bfff7640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-44 : miss 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc2bfff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc2bfff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc2bfff7640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-44 info.flags=0x0 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc2bfff7640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-44 to cache LRU end 2026-03-20T11:47:02.828 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc2bfff7640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-44 : miss 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.830+0000 7fc2bfff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc2bfff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc2bfff7640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-44 info.flags=0x0 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc2bfff7640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-44 to cache LRU end 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc2bfff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4): adding source pipe: {s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc2bfff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4): adding dest pipe: {s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc2bfff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc2bfff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc2bfff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbff95d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbff95d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 27f89a1bf404caa0fbd0e6b9ae9c8397e7076fdb4082ebd63503e57ad931fea6 2026-03-20T11:47:02.829 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.830 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.830 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.830 INFO:teuthology.orchestra.run.vm07.stderr:27f89a1bf404caa0fbd0e6b9ae9c8397e7076fdb4082ebd63503e57ad931fea6 2026-03-20T11:47:02.830 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.830 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.830 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.830 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.830 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = a0d2b6a984a87c9b8334415971896d6d914323a677e86f80000b733ee3d4c067 2026-03-20T11:47:02.830 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a0d2b6a984a87c9b8334415971896d6d914323a677e86f80000b733ee3d4c067 2026-03-20T11:47:02.830 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.830 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.830 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.830 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=35, curl_handle=0x5560dbfa5a90 2026-03-20T11:47:02.830 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc30894f900 20 run: stack=0x5560dbfe1630 is io blocked 2026-03-20T11:47:02.830 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.831+0000 7fc263fff640 20 link_request req_data=0x7fc2a8005ed0 req_data->id=35, curl_handle=0x5560dbfa5a90 2026-03-20T11:47:02.830 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.831+0000 7f3e4286b640 -1 req 13105905174658062799 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf987e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf987e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 15 stack 0x5560dbfe1440 end 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 run: stack=0x5560dbfe1440 is done 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 collect(): s=0x5560dbf8ebb0 stack=0x5560dbfe1440 is complete 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 15 stack 0x5560dbf8ebb0 end 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is done 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 4 starting trim on bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfddd50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfddd50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 enqueued request req=0x5560dbfe0910 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 req: 0x5560dbfe0910 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is io blocked 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc2beff5640 20 dequeued request req=0x5560dbfe0910 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc2beff5640 20 RGWWQ: empty 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc2beff5640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 : miss 2026-03-20T11:47:02.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc2beff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.860 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc2beff5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.860 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc2beff5640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 info.flags=0x17 2026-03-20T11:47:02.860 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc2beff5640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 to cache LRU end 2026-03-20T11:47:02.860 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc2beff5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.860 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc2beff5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.860 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc2beff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:47:02.860 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc2beff5640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-17 : miss 2026-03-20T11:47:02.860 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc2beff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.860 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc2beff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.860 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc2beff5640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-17 info.flags=0x0 2026-03-20T11:47:02.860 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc2beff5640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-17 to cache LRU end 2026-03-20T11:47:02.860 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc2beff5640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-17 : miss 2026-03-20T11:47:02.860 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc2beff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.860 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc2beff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.860 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc2beff5640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-17 info.flags=0x0 2026-03-20T11:47:02.860 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc2beff5640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-17 to cache LRU end 2026-03-20T11:47:02.860 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc2beff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22): adding source pipe: {s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.860 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc2beff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22): adding dest pipe: {s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.860 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc2beff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.860 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc2beff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.860 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc2beff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:47:02.860 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfddd50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.860 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfddd50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.860 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfddd50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.860 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfddd50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf987e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = bb478d2784b43be89e06612be9a907f430ee811bd6ea33bc6726077b65a53ecb 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:bb478d2784b43be89e06612be9a907f430ee811bd6ea33bc6726077b65a53ecb 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 6df3a99b05bd0157e4b1888d8d2eb3b38e1723308f1d4797cec2e6046d63e257 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=6df3a99b05bd0157e4b1888d8d2eb3b38e1723308f1d4797cec2e6046d63e257 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.861+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.862+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=36, curl_handle=0x5560dbf99fc0 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.862+0000 7fc30894f900 20 run: stack=0x5560dbfe1440 is io blocked 2026-03-20T11:47:02.861 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.862+0000 7fc263fff640 20 link_request req_data=0x5560dbf9aa80 req_data->id=36, curl_handle=0x5560dbf99fc0 2026-03-20T11:47:02.861 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.862+0000 7f3e2202a640 -1 req 3061691565682173941 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc30894f900 15 stack 0x5560dbfa25b0 end 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc30894f900 20 run: stack=0x5560dbfa25b0 is done 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc30894f900 20 collect(): s=0x5560dbe56840 stack=0x5560dbfa25b0 is complete 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc30894f900 15 stack 0x5560dbe56840 end 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is done 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc30894f900 4 starting trim on bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc30894f900 20 enqueued request req=0x5560dc0028b0 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc30894f900 20 req: 0x5560dc0028b0 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is io blocked 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc2be7f4640 20 dequeued request req=0x5560dc0028b0 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc2be7f4640 20 RGWWQ: empty 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc2be7f4640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 : miss 2026-03-20T11:47:02.862 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc2be7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc2be7f4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:47:02.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc2be7f4640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 info.flags=0x17 2026-03-20T11:47:02.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc2be7f4640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 to cache LRU end 2026-03-20T11:47:02.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc2be7f4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc2be7f4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc2be7f4640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:47:02.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc2be7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:47:02.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc2be7f4640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-46 : miss 2026-03-20T11:47:02.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc2be7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc2be7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc2be7f4640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-46 info.flags=0x0 2026-03-20T11:47:02.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc2be7f4640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-46 to cache LRU end 2026-03-20T11:47:02.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc2be7f4640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-46 : miss 2026-03-20T11:47:02.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.864+0000 7fc2be7f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc2be7f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc2be7f4640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-46 info.flags=0x0 2026-03-20T11:47:02.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc2be7f4640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-46 to cache LRU end 2026-03-20T11:47:02.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc2be7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6): adding source pipe: {s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc2be7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6): adding dest pipe: {s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc2be7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc2be7f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc2be7f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f6b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = f3b70ecee4a79f79d89e4bec6bb66a8d865c8113c39a17d5b5b211510608dbef 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:f3b70ecee4a79f79d89e4bec6bb66a8d865c8113c39a17d5b5b211510608dbef 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 270e8532548bbc12f9924206adf6842b1a0a09bdc28891b768b1a25990d82210 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=270e8532548bbc12f9924206adf6842b1a0a09bdc28891b768b1a25990d82210 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.864 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.865 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.865 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=37, curl_handle=0x5560dbfa2840 2026-03-20T11:47:02.865 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc30894f900 20 run: stack=0x5560dbfa25b0 is io blocked 2026-03-20T11:47:02.865 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.865+0000 7fc263fff640 20 link_request req_data=0x5560dbfa2f90 req_data->id=37, curl_handle=0x5560dbfa2840 2026-03-20T11:47:02.865 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.865+0000 7f3e48076640 -1 req 13755381613730294205 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfad4e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfad4e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfa03e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc30894f900 15 stack 0x5560dbfdd1a0 end 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc30894f900 20 run: stack=0x5560dbfdd1a0 is done 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc30894f900 20 collect(): s=0x5560dbf8e630 stack=0x5560dbfdd1a0 is complete 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc30894f900 15 stack 0x5560dbf8e630 end 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is done 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc30894f900 4 starting trim on bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfe3ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfe3ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc30894f900 20 enqueued request req=0x5560dbf9a4f0 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc30894f900 20 req: 0x5560dbf9a4f0 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc2bd7f2640 20 dequeued request req=0x5560dbf9a4f0 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc2bd7f2640 20 RGWWQ: empty 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc2bd7f2640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 : miss 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc2bd7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is io blocked 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc2bd7f2640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc2bd7f2640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 info.flags=0x17 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc2bd7f2640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 to cache LRU end 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc2bd7f2640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc2bd7f2640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc2bd7f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:47:02.867 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc2bd7f2640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-37 : miss 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc2bd7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc2bd7f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc2bd7f2640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-37 info.flags=0x0 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc2bd7f2640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-37 to cache LRU end 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc2bd7f2640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-37 : miss 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.869+0000 7fc2bd7f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc2bd7f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc2bd7f2640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-37 info.flags=0x0 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc2bd7f2640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-37 to cache LRU end 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc2bd7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42): adding source pipe: {s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc2bd7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42): adding dest pipe: {s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc2bd7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc2bd7f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc2bd7f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfe3ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfe3ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfe3ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfe3ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfad490:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfa03e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfa03e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 6d2d3086f68066b04f932cb58bba2d59ea1c3172ff1c789a59f277fffd96ab6e 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.868 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.869 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.869 INFO:teuthology.orchestra.run.vm07.stderr:6d2d3086f68066b04f932cb58bba2d59ea1c3172ff1c789a59f277fffd96ab6e 2026-03-20T11:47:02.869 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.869 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.869 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.869 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.869 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 4d324939b8621818037e4c2ae6c3735512f991c47d5574c44a6ac2f5e18e4312 2026-03-20T11:47:02.869 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=4d324939b8621818037e4c2ae6c3735512f991c47d5574c44a6ac2f5e18e4312 2026-03-20T11:47:02.869 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.869 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.869 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.869 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=38, curl_handle=0x5560dbf9d060 2026-03-20T11:47:02.869 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc30894f900 20 run: stack=0x5560dbfdd1a0 is io blocked 2026-03-20T11:47:02.869 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.870+0000 7fc263fff640 20 link_request req_data=0x7fc2a4005f20 req_data->id=38, curl_handle=0x5560dbf9d060 2026-03-20T11:47:02.869 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.870+0000 7f3e2482f640 -1 req 17661263067955053931 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.870 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbff95d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.870 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbff95d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.870 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.870 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 15 stack 0x5560dbfe1630 end 2026-03-20T11:47:02.870 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 run: stack=0x5560dbfe1630 is done 2026-03-20T11:47:02.870 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.870 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 collect(): s=0x5560dbf950e0 stack=0x5560dbfe1630 is complete 2026-03-20T11:47:02.870 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.870 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.870 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.870 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.870 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 15 stack 0x5560dbf950e0 end 2026-03-20T11:47:02.870 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.870 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is done 2026-03-20T11:47:02.870 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.870 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.870 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.870 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.870 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 4 starting trim on bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:47:02.870 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.870 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.870 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 enqueued request req=0x5560dbfdd670 2026-03-20T11:47:02.870 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.870 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 req: 0x5560dbfdd670 2026-03-20T11:47:02.870 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is io blocked 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc2bcff1640 20 dequeued request req=0x5560dbfdd670 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc2bcff1640 20 RGWWQ: empty 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc2bcff1640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 : miss 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc2bcff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc2bcff1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc2bcff1640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 info.flags=0x17 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc2bcff1640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 to cache LRU end 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc2bcff1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc2bcff1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc2bcff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc2bcff1640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-1 : miss 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc2bcff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc2bcff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc2bcff1640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-1 info.flags=0x0 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc2bcff1640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-1 to cache LRU end 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc2bcff1640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-1 : miss 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc2bcff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc2bcff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc2bcff1640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-1 info.flags=0x0 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc2bcff1640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-1 to cache LRU end 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc2bcff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1): adding source pipe: {s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc2bcff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1): adding dest pipe: {s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc2bcff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc2bcff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc2bcff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbff95d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.871 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.872+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.873+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.873+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.873+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.873+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.873+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.873+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 98f436242f091183fe8a7a8ff24e781a4a3bfb36832d1a989dc7487f6d5d652d 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.873+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:98f436242f091183fe8a7a8ff24e781a4a3bfb36832d1a989dc7487f6d5d652d 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.873+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.873+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.873+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.873+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.873+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = d8be5e011b7553ead15b788248087d24a2fd6bafde9b130f1cc6f6da637c0c0b 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.873+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=d8be5e011b7553ead15b788248087d24a2fd6bafde9b130f1cc6f6da637c0c0b 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.873+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.873+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.873+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.873+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=39, curl_handle=0x5560dbfa5a90 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.873+0000 7fc30894f900 20 run: stack=0x5560dbfe1630 is io blocked 2026-03-20T11:47:02.872 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.873+0000 7fc263fff640 20 link_request req_data=0x7fc2a8005ed0 req_data->id=39, curl_handle=0x5560dbfa5a90 2026-03-20T11:47:02.872 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.873+0000 7f3eba95b640 -1 req 10572819247773886988 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf987e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc30894f900 15 stack 0x5560dbfe1440 end 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc30894f900 20 run: stack=0x5560dbfe1440 is done 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc30894f900 20 collect(): s=0x5560dbf8ebb0 stack=0x5560dbfe1440 is complete 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc30894f900 15 stack 0x5560dbf8ebb0 end 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is done 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc30894f900 4 starting trim on bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfddd50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfddd50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc30894f900 20 enqueued request req=0x5560dbfe0910 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc30894f900 20 req: 0x5560dbfe0910 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is io blocked 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc2bc7f0640 20 dequeued request req=0x5560dbfe0910 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc2bc7f0640 20 RGWWQ: empty 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc2bc7f0640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 : miss 2026-03-20T11:47:02.901 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc2bc7f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc2bc7f0640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc2bc7f0640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 info.flags=0x17 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc2bc7f0640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 to cache LRU end 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc2bc7f0640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc2bc7f0640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc2bc7f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc2bc7f0640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-7 : miss 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc2bc7f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc2bc7f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc2bc7f0640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-7 info.flags=0x0 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc2bc7f0640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-7 to cache LRU end 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc2bc7f0640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-7 : miss 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.903+0000 7fc2bc7f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc2bc7f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc2bc7f0640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-7 info.flags=0x0 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc2bc7f0640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-7 to cache LRU end 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc2bc7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12): adding source pipe: {s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc2bc7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12): adding dest pipe: {s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc2bc7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc2bc7f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc2bc7f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfddd50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfddd50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfddd50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfddd50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf987e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf987e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.902 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 45544cd01e983f2657cd5b3a8cbc25e1cb6d3d4cfa81b821190fd3e8017c9dd4 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:45544cd01e983f2657cd5b3a8cbc25e1cb6d3d4cfa81b821190fd3e8017c9dd4 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 3d8fd7a191ace0ea0bc8e961ea25f359433b06409315eccc19791a245544555c 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=3d8fd7a191ace0ea0bc8e961ea25f359433b06409315eccc19791a245544555c 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=40, curl_handle=0x5560dbf99fc0 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc30894f900 20 run: stack=0x5560dbfe1440 is io blocked 2026-03-20T11:47:02.903 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.904+0000 7fc263fff640 20 link_request req_data=0x5560dbf9aa80 req_data->id=40, curl_handle=0x5560dbf99fc0 2026-03-20T11:47:02.903 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.904+0000 7f3ebb95d640 -1 req 6028173464697211824 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f6b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc30894f900 15 stack 0x5560dbfa25b0 end 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc30894f900 20 run: stack=0x5560dbfa25b0 is done 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc30894f900 20 collect(): s=0x5560dbe56840 stack=0x5560dbfa25b0 is complete 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc30894f900 15 stack 0x5560dbe56840 end 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is done 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc30894f900 4 starting trim on bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc30894f900 20 enqueued request req=0x5560dc0028b0 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc30894f900 20 req: 0x5560dc0028b0 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is io blocked 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc2bbfef640 20 dequeued request req=0x5560dc0028b0 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc2bbfef640 20 RGWWQ: empty 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc2bbfef640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 : miss 2026-03-20T11:47:02.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.906+0000 7fc2bbfef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.905 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.907+0000 7fc2bbfef640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:47:02.905 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.907+0000 7fc2bbfef640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 info.flags=0x17 2026-03-20T11:47:02.905 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.907+0000 7fc2bbfef640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 to cache LRU end 2026-03-20T11:47:02.905 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.907+0000 7fc2bbfef640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.905 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.907+0000 7fc2bbfef640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.905 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.907+0000 7fc2bbfef640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:47:02.905 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.907+0000 7fc2bbfef640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:47:02.905 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.907+0000 7fc2bbfef640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-48 : miss 2026-03-20T11:47:02.905 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.907+0000 7fc2bbfef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.906 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc2bbfef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.906 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc2bbfef640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-48 info.flags=0x0 2026-03-20T11:47:02.906 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc2bbfef640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-48 to cache LRU end 2026-03-20T11:47:02.906 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc2bbfef640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-48 : miss 2026-03-20T11:47:02.906 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc2bbfef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.906 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc2bbfef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.906 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc2bbfef640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-48 info.flags=0x0 2026-03-20T11:47:02.906 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc2bbfef640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-48 to cache LRU end 2026-03-20T11:47:02.906 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc2bbfef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8): adding source pipe: {s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.906 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc2bbfef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8): adding dest pipe: {s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.906 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc2bbfef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.906 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc2bbfef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.906 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc2bbfef640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:47:02.906 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.906 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 0725faf7919aa753f3a1f6e16f358d95741356c6c10a62b79761305342e757e9 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:0725faf7919aa753f3a1f6e16f358d95741356c6c10a62b79761305342e757e9 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 168983480e16980a4d58403f96e033701c791d7e1467ac2d76ddf0df980d353c 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=168983480e16980a4d58403f96e033701c791d7e1467ac2d76ddf0df980d353c 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=41, curl_handle=0x5560dbfa2840 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc30894f900 20 run: stack=0x5560dbfa25b0 is io blocked 2026-03-20T11:47:02.907 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.908+0000 7fc263fff640 20 link_request req_data=0x5560dbfa2f90 req_data->id=41, curl_handle=0x5560dbfa2840 2026-03-20T11:47:02.907 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.909+0000 7f3e3b85d640 -1 req 208263599652002821 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfa03e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfa03e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfad490:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc30894f900 15 stack 0x5560dbfdd1a0 end 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc30894f900 20 run: stack=0x5560dbfdd1a0 is done 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc30894f900 20 collect(): s=0x5560dbf8e630 stack=0x5560dbfdd1a0 is complete 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc30894f900 15 stack 0x5560dbf8e630 end 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is done 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc30894f900 4 starting trim on bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfe3ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfe3ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc30894f900 20 enqueued request req=0x5560dbf9a4f0 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc30894f900 20 req: 0x5560dbf9a4f0 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is io blocked 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc2bb7ee640 20 dequeued request req=0x5560dbf9a4f0 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc2bb7ee640 20 RGWWQ: empty 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc2bb7ee640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 : miss 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc2bb7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.909 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc2bb7ee640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc2bb7ee640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 info.flags=0x17 2026-03-20T11:47:02.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc2bb7ee640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 to cache LRU end 2026-03-20T11:47:02.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc2bb7ee640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc2bb7ee640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc2bb7ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:47:02.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc2bb7ee640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-59 : miss 2026-03-20T11:47:02.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc2bb7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc2bb7ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc2bb7ee640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-59 info.flags=0x0 2026-03-20T11:47:02.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc2bb7ee640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-59 to cache LRU end 2026-03-20T11:47:02.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc2bb7ee640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-59 : miss 2026-03-20T11:47:02.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.911+0000 7fc2bb7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc2bb7ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc2bb7ee640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-59 info.flags=0x0 2026-03-20T11:47:02.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc2bb7ee640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-59 to cache LRU end 2026-03-20T11:47:02.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc2bb7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19): adding source pipe: {s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc2bb7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19): adding dest pipe: {s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc2bb7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc2bb7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc2bb7ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:47:02.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfe3ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfe3ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfe3ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbfe3ae0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.910 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfa03e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfe3ae0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfe3ae0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 242a37594fc8f78a27bf114623e3c9e31b6c135050ea88ed1e4241a55536c49f 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:242a37594fc8f78a27bf114623e3c9e31b6c135050ea88ed1e4241a55536c49f 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = ae17aa83005cc4aa9959aa3f4c61b100398737621e6611887c3e2e8625d76840 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=ae17aa83005cc4aa9959aa3f4c61b100398737621e6611887c3e2e8625d76840 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=42, curl_handle=0x5560dbf9d060 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc30894f900 20 run: stack=0x5560dbfdd1a0 is io blocked 2026-03-20T11:47:02.911 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.912+0000 7fc263fff640 20 link_request req_data=0x7fc2a4005f20 req_data->id=42, curl_handle=0x5560dbf9d060 2026-03-20T11:47:02.911 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.913+0000 7f3e2382d640 -1 req 12377712508058877782 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbff95d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc30894f900 15 stack 0x5560dbfe1630 end 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc30894f900 20 run: stack=0x5560dbfe1630 is done 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc30894f900 20 collect(): s=0x5560dbf950e0 stack=0x5560dbfe1630 is complete 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc30894f900 15 stack 0x5560dbf950e0 end 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is done 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc30894f900 4 starting trim on bucket=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc30894f900 20 enqueued request req=0x5560dbfdd670 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc30894f900 20 req: 0x5560dbfdd670 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is io blocked 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc2bafed640 20 dequeued request req=0x5560dbfdd670 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc2bafed640 20 RGWWQ: empty 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc2bafed640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 : miss 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc2bafed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc2bafed640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc2bafed640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 info.flags=0x17 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc2bafed640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 to cache LRU end 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc2bafed640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc2bafed640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.913 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc2bafed640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc2bafed640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-6 : miss 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc2bafed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc2bafed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc2bafed640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-6 info.flags=0x0 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc2bafed640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-6 to cache LRU end 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc2bafed640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-6 : miss 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.915+0000 7fc2bafed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc2bafed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc2bafed640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-6 info.flags=0x0 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc2bafed640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-6 to cache LRU end 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc2bafed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11): adding source pipe: {s={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc2bafed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11): adding dest pipe: {s={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc2bafed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc2bafed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc2bafed640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbff95a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbff95a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.914 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.915 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.915 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.915 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.915 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.915 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.915 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.915 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.915 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.915 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.915 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.915 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.915 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 315ddc28ec211c2bba496fdd53af7e300070b2b0b5eab0d105ed75048fb59441 2026-03-20T11:47:02.915 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.915 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.915 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.915 INFO:teuthology.orchestra.run.vm07.stderr:315ddc28ec211c2bba496fdd53af7e300070b2b0b5eab0d105ed75048fb59441 2026-03-20T11:47:02.915 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.915 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.915 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.915 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.915 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = c5d5c817baf837613a55b33c2ac38b467a64cb7bbc003e4b87a226bcbadf4181 2026-03-20T11:47:02.915 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c5d5c817baf837613a55b33c2ac38b467a64cb7bbc003e4b87a226bcbadf4181 2026-03-20T11:47:02.915 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.915 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.915 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.915 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=43, curl_handle=0x5560dbfa5a90 2026-03-20T11:47:02.915 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc30894f900 20 run: stack=0x5560dbfe1630 is io blocked 2026-03-20T11:47:02.915 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.916+0000 7fc263fff640 20 link_request req_data=0x7fc2a8005ed0 req_data->id=43, curl_handle=0x5560dbfa5a90 2026-03-20T11:47:02.915 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.917+0000 7f3e2302c640 -1 req 17571445029743079801 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf987e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf987e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc30894f900 15 stack 0x5560dbfe1440 end 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc30894f900 20 run: stack=0x5560dbfe1440 is done 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc30894f900 20 collect(): s=0x5560dbf8ebb0 stack=0x5560dbfe1440 is complete 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c70:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc30894f900 15 stack 0x5560dbf8ebb0 end 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is done 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc30894f900 4 starting trim on bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfddd50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfddd50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc30894f900 20 enqueued request req=0x5560dbfe0910 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc30894f900 20 req: 0x5560dbfe0910 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is io blocked 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc2ba7ec640 20 dequeued request req=0x5560dbfe0910 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc2ba7ec640 20 RGWWQ: empty 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc2ba7ec640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 : miss 2026-03-20T11:47:02.943 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc2ba7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.944 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc2ba7ec640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.944 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc2ba7ec640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 info.flags=0x17 2026-03-20T11:47:02.944 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc2ba7ec640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 to cache LRU end 2026-03-20T11:47:02.944 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc2ba7ec640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.944 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc2ba7ec640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.944 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc2ba7ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:47:02.944 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc2ba7ec640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-21 : miss 2026-03-20T11:47:02.944 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc2ba7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.944 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc2ba7ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.944 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc2ba7ec640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-21 info.flags=0x0 2026-03-20T11:47:02.944 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc2ba7ec640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-21 to cache LRU end 2026-03-20T11:47:02.944 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc2ba7ec640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-21 : miss 2026-03-20T11:47:02.944 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.945+0000 7fc2ba7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.944 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.946+0000 7f3eb214a640 -1 req 9751724624937012325 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.944 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc2ba7ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.944 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc2ba7ec640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-21 info.flags=0x0 2026-03-20T11:47:02.944 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc2ba7ec640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-21 to cache LRU end 2026-03-20T11:47:02.944 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc2ba7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26): adding source pipe: {s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc2ba7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26): adding dest pipe: {s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc2ba7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc2ba7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc2ba7ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfddd50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfddd50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfddd50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfddd50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf987e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 5d533174c286933f06192b611c0528cc106659c13111b7c3e74ee28c502d09ac 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:5d533174c286933f06192b611c0528cc106659c13111b7c3e74ee28c502d09ac 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.945 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.946 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.946 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.946 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 0f5c1aebcd9f6f533d135beb8cdc27fb729ba9bb62b33ad9b58791ce6982a879 2026-03-20T11:47:02.946 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=0f5c1aebcd9f6f533d135beb8cdc27fb729ba9bb62b33ad9b58791ce6982a879 2026-03-20T11:47:02.946 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.946 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.946 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.946 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=44, curl_handle=0x5560dbf99fc0 2026-03-20T11:47:02.946 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc30894f900 20 run: stack=0x5560dbfe1440 is io blocked 2026-03-20T11:47:02.946 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.946+0000 7fc263fff640 20 link_request req_data=0x5560dbf9aa80 req_data->id=44, curl_handle=0x5560dbf99fc0 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f6b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc30894f900 15 stack 0x5560dbfa25b0 end 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc30894f900 20 run: stack=0x5560dbfa25b0 is done 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc30894f900 20 collect(): s=0x5560dbe56840 stack=0x5560dbfa25b0 is complete 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc30894f900 15 stack 0x5560dbe56840 end 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is done 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc30894f900 4 starting trim on bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc30894f900 20 enqueued request req=0x5560dc0028b0 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc30894f900 20 req: 0x5560dc0028b0 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is io blocked 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc2b9feb640 20 dequeued request req=0x5560dc0028b0 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc2b9feb640 20 RGWWQ: empty 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc2b9feb640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 : miss 2026-03-20T11:47:02.947 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc2b9feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc2b9feb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc2b9feb640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 info.flags=0x17 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc2b9feb640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 to cache LRU end 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc2b9feb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc2b9feb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc2b9feb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc2b9feb640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-41 : miss 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc2b9feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc2b9feb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc2b9feb640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-41 info.flags=0x0 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc2b9feb640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-41 to cache LRU end 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc2b9feb640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-41 : miss 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc2b9feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc2b9feb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc2b9feb640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-41 info.flags=0x0 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc2b9feb640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-41 to cache LRU end 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.949+0000 7fc2b9feb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1): adding source pipe: {s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc2b9feb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1): adding dest pipe: {s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc2b9feb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc2b9feb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc2b9feb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f6b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.948 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.949 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.949 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.949 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.949 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.949 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.949 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.949 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.949 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = f238a505a9fd93e1298a3bf89218f066a9397c594bfb8572a815e5bf7dc97da0 2026-03-20T11:47:02.949 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.949 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.949 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.949 INFO:teuthology.orchestra.run.vm07.stderr:f238a505a9fd93e1298a3bf89218f066a9397c594bfb8572a815e5bf7dc97da0 2026-03-20T11:47:02.949 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.949 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.949 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.949 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.949 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = ba033f5879f6a27c08751715ebbf405e4cb8adb9a3f0a6a88cce1077620dff28 2026-03-20T11:47:02.949 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=ba033f5879f6a27c08751715ebbf405e4cb8adb9a3f0a6a88cce1077620dff28 2026-03-20T11:47:02.949 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.949 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.949 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.949 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=45, curl_handle=0x5560dbfa2840 2026-03-20T11:47:02.949 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc30894f900 20 run: stack=0x5560dbfa25b0 is io blocked 2026-03-20T11:47:02.949 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.950+0000 7fc263fff640 20 link_request req_data=0x5560dbfa2f90 req_data->id=45, curl_handle=0x5560dbfa2840 2026-03-20T11:47:02.949 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.950+0000 7f3e26833640 -1 req 9293724211829522082 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfe3ae0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfe3ae0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfa03e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc30894f900 15 stack 0x5560dbfdd1a0 end 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc30894f900 20 run: stack=0x5560dbfdd1a0 is done 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc30894f900 20 collect(): s=0x5560dbf8e630 stack=0x5560dbfdd1a0 is complete 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc30894f900 15 stack 0x5560dbf8e630 end 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is done 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc30894f900 4 starting trim on bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dc001dc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dc001dc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc30894f900 20 enqueued request req=0x5560dbf9a4f0 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc30894f900 20 req: 0x5560dbf9a4f0 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is io blocked 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc2b97ea640 20 dequeued request req=0x5560dbf9a4f0 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc2b97ea640 20 RGWWQ: empty 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc2b97ea640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 : miss 2026-03-20T11:47:02.952 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc2b97ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc2b97ea640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc2b97ea640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 info.flags=0x17 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc2b97ea640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 to cache LRU end 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc2b97ea640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc2b97ea640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc2b97ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc2b97ea640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-49 : miss 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc2b97ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc2b97ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc2b97ea640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-49 info.flags=0x0 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc2b97ea640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-49 to cache LRU end 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc2b97ea640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-49 : miss 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc2b97ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc2b97ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc2b97ea640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-49 info.flags=0x0 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc2b97ea640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-49 to cache LRU end 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc2b97ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9): adding source pipe: {s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc2b97ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9): adding dest pipe: {s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc2b97ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc2b97ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc2b97ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dc001dc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dc001dc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.954+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dc001dc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.955+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dc001dc0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.955+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.955+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dc001dc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.955+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.955+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.955+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfda9d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.955+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfda9d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.955+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.955+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.955+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.955+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.955+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.955+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.953 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.955+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 9ebabdfbf22aa4c9116857186e8253c5d79754df7b14ea2e9d89a45d06c49175 2026-03-20T11:47:02.954 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.955+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.954 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.954 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.954 INFO:teuthology.orchestra.run.vm07.stderr:9ebabdfbf22aa4c9116857186e8253c5d79754df7b14ea2e9d89a45d06c49175 2026-03-20T11:47:02.954 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.955+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.954 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.955+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.954 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.955+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.954 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.955+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.954 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.955+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = 856d6c098889805f8e8cb5f83ea5d979b630dd9bea5fe3dca8c1ce4444968084 2026-03-20T11:47:02.954 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.955+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=856d6c098889805f8e8cb5f83ea5d979b630dd9bea5fe3dca8c1ce4444968084 2026-03-20T11:47:02.954 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.955+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.954 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.955+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.954 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.955+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.954 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.955+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=46, curl_handle=0x5560dbf9d060 2026-03-20T11:47:02.954 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.955+0000 7fc30894f900 20 run: stack=0x5560dbfdd1a0 is io blocked 2026-03-20T11:47:02.954 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.955+0000 7fc263fff640 20 link_request req_data=0x7fc2a4005f20 req_data->id=46, curl_handle=0x5560dbf9d060 2026-03-20T11:47:02.954 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.955+0000 7f3e99919640 -1 req 10848068431037624570 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbff95a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbff95a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc30894f900 15 stack 0x5560dbfe1630 end 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc30894f900 20 run: stack=0x5560dbfe1630 is done 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc30894f900 20 collect(): s=0x5560dbf950e0 stack=0x5560dbfe1630 is complete 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc30894f900 15 stack 0x5560dbf950e0 end 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is done 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc30894f900 4 starting trim on bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc30894f900 20 enqueued request req=0x5560dbfdd670 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc30894f900 20 req: 0x5560dbfdd670 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is io blocked 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc2b8fe9640 20 dequeued request req=0x5560dbfdd670 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc2b8fe9640 20 RGWWQ: empty 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc2b8fe9640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 : miss 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc2b8fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc2b8fe9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc2b8fe9640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 info.flags=0x17 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc2b8fe9640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 to cache LRU end 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc2b8fe9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc2b8fe9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc2b8fe9640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc2b8fe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc2b8fe9640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-45 : miss 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc2b8fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc2b8fe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc2b8fe9640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-45 info.flags=0x0 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc2b8fe9640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-45 to cache LRU end 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc2b8fe9640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-45 : miss 2026-03-20T11:47:02.955 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc2b8fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc2b8fe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc2b8fe9640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-45 info.flags=0x0 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc2b8fe9640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-45 to cache LRU end 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc2b8fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5): adding source pipe: {s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc2b8fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5): adding dest pipe: {s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc2b8fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc2b8fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.957+0000 7fc2b8fe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dc004f40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbff9560:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbff9560:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 84040295e83cbc3ddfd9835cbb0e94d6ddee2ea23a00eba99a39b2355c7fe30e 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:84040295e83cbc3ddfd9835cbb0e94d6ddee2ea23a00eba99a39b2355c7fe30e 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = df0ed646071868859bda6e12091b150748361a19d9807447167c03c2535983cd 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=df0ed646071868859bda6e12091b150748361a19d9807447167c03c2535983cd 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.956 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.957 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.957 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=47, curl_handle=0x5560dbfa5a90 2026-03-20T11:47:02.957 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc30894f900 20 run: stack=0x5560dbfe1630 is io blocked 2026-03-20T11:47:02.957 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.958+0000 7fc263fff640 20 link_request req_data=0x7fc2a8005ed0 req_data->id=47, curl_handle=0x5560dbfa5a90 2026-03-20T11:47:02.957 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.958+0000 7f3e56092640 -1 req 1034315527567388363 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf987e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc30894f900 15 stack 0x5560dbfe1440 end 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc30894f900 20 run: stack=0x5560dbfe1440 is done 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc30894f900 20 collect(): s=0x5560dbf8ebb0 stack=0x5560dbfe1440 is complete 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c00:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc30894f900 15 stack 0x5560dbf8ebb0 end 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is done 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc30894f900 4 starting trim on bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfddd50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfddd50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc30894f900 20 enqueued request req=0x5560dbfe0910 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc30894f900 20 req: 0x5560dbfe0910 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is io blocked 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc2b87e8640 20 dequeued request req=0x5560dbfe0910 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc2b87e8640 20 RGWWQ: empty 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc2b87e8640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 : miss 2026-03-20T11:47:02.985 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc2b87e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc2b87e8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc2b87e8640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 info.flags=0x17 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc2b87e8640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 to cache LRU end 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc2b87e8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc2b87e8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc2b87e8640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc2b87e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc2b87e8640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-33 : miss 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc2b87e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc2b87e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc2b87e8640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-33 info.flags=0x0 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc2b87e8640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-33 to cache LRU end 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc2b87e8640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-33 : miss 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc2b87e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc2b87e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc2b87e8640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-33 info.flags=0x0 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc2b87e8640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-33 to cache LRU end 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc2b87e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38): adding source pipe: {s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.987+0000 7fc2b87e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38): adding dest pipe: {s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc2b87e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc2b87e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc2b87e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfddd50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfddd50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfddd50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbfddd50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf987e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.986 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = 701135ccad112d9def97ec2e632b37b530ace590af476f8e8c22f55d98b93b21 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:701135ccad112d9def97ec2e632b37b530ace590af476f8e8c22f55d98b93b21 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = f60d6181096dc4e59affe8e7d5bc037d8c69eea72fc65a2857dde09afb678ac0 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=f60d6181096dc4e59affe8e7d5bc037d8c69eea72fc65a2857dde09afb678ac0 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=48, curl_handle=0x5560dbf99fc0 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc30894f900 20 run: stack=0x5560dbfe1440 is io blocked 2026-03-20T11:47:02.987 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.988+0000 7fc263fff640 20 link_request req_data=0x5560dbf9aa80 req_data->id=48, curl_handle=0x5560dbf99fc0 2026-03-20T11:47:02.987 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.988+0000 7f3e3304c640 -1 req 2911481562677228745 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.989 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.989 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.989 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f6b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.989 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc30894f900 15 stack 0x5560dbfa25b0 end 2026-03-20T11:47:02.989 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc30894f900 20 run: stack=0x5560dbfa25b0 is done 2026-03-20T11:47:02.989 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.989 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc30894f900 20 collect(): s=0x5560dbe56840 stack=0x5560dbfa25b0 is complete 2026-03-20T11:47:02.989 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.989 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.989 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.989 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.989 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc30894f900 15 stack 0x5560dbe56840 end 2026-03-20T11:47:02.989 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.989 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is done 2026-03-20T11:47:02.989 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.989 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.989 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.989 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.989 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc30894f900 4 starting trim on bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:47:02.989 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.989 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.989 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc30894f900 20 enqueued request req=0x5560dc0028b0 2026-03-20T11:47:02.989 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc30894f900 20 RGWWQ: 2026-03-20T11:47:02.989 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc30894f900 20 req: 0x5560dc0028b0 2026-03-20T11:47:02.989 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is io blocked 2026-03-20T11:47:02.989 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc2b7fe7640 20 dequeued request req=0x5560dc0028b0 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc2b7fe7640 20 RGWWQ: empty 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc2b7fe7640 10 rgw async rados processor: cache get: name=b1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 : miss 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc2b7fe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc2b7fe7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc2b7fe7640 10 rgw async rados processor: cache put: name=b1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 info.flags=0x17 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc2b7fe7640 10 rgw async rados processor: adding b1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 to cache LRU end 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc2b7fe7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc2b7fe7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc2b7fe7640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc2b7fe7640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc2b7fe7640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-source-hints.dipepu-47 : miss 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc2b7fe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc2b7fe7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc2b7fe7640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-source-hints.dipepu-47 info.flags=0x0 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc2b7fe7640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-source-hints.dipepu-47 to cache LRU end 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc2b7fe7640 10 rgw async rados processor: cache get: name=b1.rgw.log++bucket.sync-target-hints.dipepu-47 : miss 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.991+0000 7fc2b7fe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc2b7fe7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc2b7fe7640 10 rgw async rados processor: cache put: name=b1.rgw.log++bucket.sync-target-hints.dipepu-47 info.flags=0x0 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc2b7fe7640 10 rgw async rados processor: adding b1.rgw.log++bucket.sync-target-hints.dipepu-47 to cache LRU end 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc2b7fe7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7): adding source pipe: {s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc2b7fe7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7): adding dest pipe: {s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc2b7fe7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc2b7fe7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc2b7fe7640 10 rgw async rados processor: chain_cache_entry: cache_locator=b1.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf97b50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f6b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.990 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 20 get_url picked endpoint=http://vm08.local:8001 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm08.local:8001 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&options=merge&rgwx-zonegroup=b&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&status=&type=bucket-index&version=2 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:02 +0000 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:host:vm08.local:8001 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114702Z 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]canonical request hash = fd1bbdd272c9338c9b2a83b87f8be68f9bac8b7df57f7b700b86ab055ef91b5b 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114702Z 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:fd1bbdd272c9338c9b2a83b87f8be68f9bac8b7df57f7b700b86ab055ef91b5b 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 10 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]generated signature = c05914c10e87d3ac42e4a04ec716fa833783d0268708a2d88a2e9586095eb967 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c05914c10e87d3ac42e4a04ec716fa833783d0268708a2d88a2e9586095eb967 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 20 http_client[GET/http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114702Z 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 20 sending request to http://vm08.local:8001/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c&version=2&rgwx-zonegroup=b 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 20 register_request mgr=0x7ffd994db390 req_data->id=49, curl_handle=0x5560dbfa2840 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc30894f900 20 run: stack=0x5560dbfa25b0 is io blocked 2026-03-20T11:47:02.991 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.992+0000 7fc263fff640 20 link_request req_data=0x5560dbfa2f90 req_data->id=49, curl_handle=0x5560dbfa2840 2026-03-20T11:47:02.991 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:47:02.992+0000 7f3e28036640 -1 req 3777966691401698712 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} returned ret=-2 2026-03-20T11:47:02.994 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.996+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfda9d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.994 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.996+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dbfda9d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.994 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.996+0000 7fc30894f900 20 cr:s=0x5560dbfdd1a0:op=0x5560dc001dc0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.994 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.996+0000 7fc30894f900 15 stack 0x5560dbfdd1a0 end 2026-03-20T11:47:02.994 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.996+0000 7fc30894f900 20 run: stack=0x5560dbfdd1a0 is done 2026-03-20T11:47:02.994 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.996+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.994 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.996+0000 7fc30894f900 20 collect(): s=0x5560dbf8e630 stack=0x5560dbfdd1a0 is complete 2026-03-20T11:47:02.994 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.996+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.994 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.996+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.994 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.996+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.994 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.996+0000 7fc30894f900 20 cr:s=0x5560dbf8e630:op=0x5560dbf92f40:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.994 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.996+0000 7fc30894f900 15 stack 0x5560dbf8e630 end 2026-03-20T11:47:02.994 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.996+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.994 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.996+0000 7fc30894f900 20 run: stack=0x5560dbf8e630 is done 2026-03-20T11:47:02.994 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.996+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.994 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.996+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.994 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.996+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:02.997 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.999+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbff9560:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:02.997 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.999+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbff9560:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:02.997 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.999+0000 7fc30894f900 20 cr:s=0x5560dbfe1630:op=0x5560dbffa530:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:02.997 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.999+0000 7fc30894f900 15 stack 0x5560dbfe1630 end 2026-03-20T11:47:02.997 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.999+0000 7fc30894f900 20 run: stack=0x5560dbfe1630 is done 2026-03-20T11:47:02.997 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.999+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:02.997 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.999+0000 7fc30894f900 20 collect(): s=0x5560dbf950e0 stack=0x5560dbfe1630 is complete 2026-03-20T11:47:02.997 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.999+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:02.997 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.999+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:02.997 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.999+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:02.997 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.999+0000 7fc30894f900 20 cr:s=0x5560dbf950e0:op=0x5560dbf8d0c0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:02.997 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.999+0000 7fc30894f900 15 stack 0x5560dbf950e0 end 2026-03-20T11:47:02.997 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.999+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:02.997 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.999+0000 7fc30894f900 20 run: stack=0x5560dbf950e0 is done 2026-03-20T11:47:02.997 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.999+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:02.997 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.999+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:02.997 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:02.999+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:03.027 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.029+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:03.027 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.029+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf94210:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:03.027 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.029+0000 7fc30894f900 20 cr:s=0x5560dbfe1440:op=0x5560dbf987e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:03.027 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.029+0000 7fc30894f900 15 stack 0x5560dbfe1440 end 2026-03-20T11:47:03.027 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.029+0000 7fc30894f900 20 run: stack=0x5560dbfe1440 is done 2026-03-20T11:47:03.027 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.029+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c00:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:03.027 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.029+0000 7fc30894f900 20 collect(): s=0x5560dbf8ebb0 stack=0x5560dbfe1440 is complete 2026-03-20T11:47:03.027 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.029+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:03.027 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.029+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:03.027 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.029+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:03.027 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.029+0000 7fc30894f900 20 cr:s=0x5560dbf8ebb0:op=0x5560dbf91c00:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:03.027 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.029+0000 7fc30894f900 15 stack 0x5560dbf8ebb0 end 2026-03-20T11:47:03.027 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.029+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:03.027 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.029+0000 7fc30894f900 20 run: stack=0x5560dbf8ebb0 is done 2026-03-20T11:47:03.027 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.029+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:03.027 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.029+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:03.027 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.029+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:03.031 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.033+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:03.031 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.033+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf8f500:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:03.031 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.033+0000 7fc30894f900 20 cr:s=0x5560dbfa25b0:op=0x5560dbf9f6b0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:03.031 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.033+0000 7fc30894f900 15 stack 0x5560dbfa25b0 end 2026-03-20T11:47:03.031 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.033+0000 7fc30894f900 20 run: stack=0x5560dbfa25b0 is done 2026-03-20T11:47:03.031 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.033+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:03.031 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.033+0000 7fc30894f900 20 collect(): s=0x5560dbe56840 stack=0x5560dbfa25b0 is complete 2026-03-20T11:47:03.031 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.033+0000 7fc30894f900 10 min_generation is 0 2026-03-20T11:47:03.031 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.033+0000 7fc30894f900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:03.031 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.033+0000 7fc30894f900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:03.031 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.033+0000 7fc30894f900 20 cr:s=0x5560dbe56840:op=0x5560dbf90930:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:03.031 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.033+0000 7fc30894f900 15 stack 0x5560dbe56840 end 2026-03-20T11:47:03.031 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.033+0000 7fc30894f900 20 stack->operate() returned ret=-22 2026-03-20T11:47:03.031 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.033+0000 7fc30894f900 20 run: stack=0x5560dbe56840 is done 2026-03-20T11:47:03.031 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.033+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:03.031 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.033+0000 7fc30894f900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:03.031 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.033+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf90020:27BucketTrimInstanceCollectCR: operate() returned r=-22 2026-03-20T11:47:03.031 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.033+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf8b9b0:12BucketTrimCR: operate() 2026-03-20T11:47:03.031 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.033+0000 7fc30894f900 20 writing bucket trim marker=12:a04872b5:root::dipepu-33:head 2026-03-20T11:47:03.031 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.033+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dc004f40:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:03.031 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.033+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dc004f40:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:03.031 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.033+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is io blocked 2026-03-20T11:47:03.033 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.035+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dc004f40:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:03.033 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.035+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dc004f40:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:03.033 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.035+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dc004f40:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:03.033 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.035+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dc004f40:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:03.033 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.035+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf8b9b0:12BucketTrimCR: operate() 2026-03-20T11:47:03.033 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.035+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dc001dc0:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:03.033 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.035+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dc001dc0:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:03.033 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.035+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is io blocked 2026-03-20T11:47:03.033 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.035+0000 7fc2d37fe640 20 trim: bucket trim completed 2026-03-20T11:47:03.033 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.035+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dc001dc0:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:03.033 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.035+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dc001dc0:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:03.033 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.035+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dc001dc0:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:03.033 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.035+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dc001dc0:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:03.033 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.035+0000 7fc30894f900 20 cr:s=0x5560dbf8b770:op=0x5560dbf8b9b0:12BucketTrimCR: operate() 2026-03-20T11:47:03.033 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.035+0000 7fc30894f900 4 bucket index log processing completed in 0.719246149s 2026-03-20T11:47:03.033 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.035+0000 7fc30894f900 15 stack 0x5560dbf8b770 end 2026-03-20T11:47:03.033 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.035+0000 7fc30894f900 20 run: stack=0x5560dbf8b770 is done 2026-03-20T11:47:03.038 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.040+0000 7fc30894f900 20 remove_watcher() i=5 2026-03-20T11:47:03.038 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.040+0000 7fc30894f900 2 removed watcher, disabling cache 2026-03-20T11:47:03.039 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.041+0000 7fc30894f900 20 remove_watcher() i=1 2026-03-20T11:47:03.039 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.041+0000 7fc30894f900 20 remove_watcher() i=6 2026-03-20T11:47:03.039 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.041+0000 7fc30894f900 20 remove_watcher() i=4 2026-03-20T11:47:03.039 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.041+0000 7fc30894f900 20 remove_watcher() i=3 2026-03-20T11:47:03.040 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.041+0000 7fc30894f900 20 remove_watcher() i=0 2026-03-20T11:47:03.040 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.041+0000 7fc30894f900 20 remove_watcher() i=2 2026-03-20T11:47:03.040 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:03.041+0000 7fc30894f900 20 remove_watcher() i=7 2026-03-20T11:47:08.046 INFO:rgw_multi.tests:trimming on zone=b2 2026-03-20T11:47:08.046 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '50', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:47:08.046 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '50', '--rgw-zone', 'b2', '--rgw-zonegroup', 'b', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:47:08.046 DEBUG:teuthology.orchestra.run.vm07:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bilog autotrim --rgw-sync-log-trim-max-buckets 50 --rgw-zone b2 --rgw-zonegroup b --rgw-realm test-realm --cluster c2 --debug-rgw 20 --debug-ms 0 2026-03-20T11:47:08.085 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:47:08.085 INFO:teuthology.orchestra.run.vm07.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:47:08.105 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.107+0000 7fdfe23ce900 20 rados->read ofs=0 len=0 2026-03-20T11:47:08.106 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.108+0000 7fdfe23ce900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:47:08.106 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.108+0000 7fdfe23ce900 20 rados->read ofs=0 len=0 2026-03-20T11:47:08.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.109+0000 7fdfe23ce900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:47:08.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.109+0000 7fdfe23ce900 20 realm test-realm 6a789b89-c2b8-45f5-82cc-fb80234d0c6c 2026-03-20T11:47:08.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.109+0000 7fdfe23ce900 20 rados->read ofs=0 len=0 2026-03-20T11:47:08.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.109+0000 7fdfe23ce900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:47:08.107 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.109+0000 7fdfe23ce900 20 rados->read ofs=0 len=0 2026-03-20T11:47:08.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.110+0000 7fdfe23ce900 20 rados_obj.operate() r=0 bl.length=10 2026-03-20T11:47:08.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.110+0000 7fdfe23ce900 20 rados->read ofs=0 len=0 2026-03-20T11:47:08.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.110+0000 7fdfe23ce900 20 rados_obj.operate() r=0 bl.length=2020 2026-03-20T11:47:08.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.110+0000 7fdfe23ce900 20 rados->read ofs=0 len=0 2026-03-20T11:47:08.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.110+0000 7fdfe23ce900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:47:08.108 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.110+0000 7fdfe23ce900 20 rados->read ofs=0 len=0 2026-03-20T11:47:08.109 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.111+0000 7fdfe23ce900 20 rados_obj.operate() r=0 bl.length=1030 2026-03-20T11:47:08.109 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.111+0000 7fdfe23ce900 20 zone b2 found 2026-03-20T11:47:08.109 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.111+0000 7fdfe23ce900 4 Realm: test-realm (6a789b89-c2b8-45f5-82cc-fb80234d0c6c) 2026-03-20T11:47:08.109 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.111+0000 7fdfe23ce900 4 ZoneGroup: b (b) 2026-03-20T11:47:08.109 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.111+0000 7fdfe23ce900 4 Zone: b2 (cd52b229-0d4a-43bf-b4ba-156aa19e7aef) 2026-03-20T11:47:08.109 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.111+0000 7fdfe23ce900 4 using period configuration: cb2fcdce-2c1b-429a-8601-a284ee3a2381:12 2026-03-20T11:47:08.109 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.111+0000 7fdfe23ce900 20 period zonegroup name b 2026-03-20T11:47:08.109 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.111+0000 7fdfe23ce900 20 using current period zonegroup b 2026-03-20T11:47:08.109 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.111+0000 7fdfe23ce900 20 rados->read ofs=0 len=0 2026-03-20T11:47:08.109 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.111+0000 7fdfe23ce900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.109 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.111+0000 7fdfe23ce900 20 rados->read ofs=0 len=0 2026-03-20T11:47:08.110 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.112+0000 7fdfe23ce900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.110 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.112+0000 7fdfe23ce900 20 init(): pipe_map (zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:47:08.110 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.112+0000 7fdfe23ce900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.110 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.112+0000 7fdfe23ce900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.110 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.112+0000 7fdfe23ce900 20 rados->read ofs=0 len=0 2026-03-20T11:47:08.110 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.112+0000 7fdfe23ce900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.110 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.112+0000 7fdfe23ce900 20 rados->read ofs=0 len=0 2026-03-20T11:47:08.110 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.112+0000 7fdfe23ce900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.112+0000 7fdfe23ce900 20 init(): pipe_map (zone=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:47:08.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.112+0000 7fdfe23ce900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.112+0000 7fdfe23ce900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.112+0000 7fdfe23ce900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.112+0000 7fdfe23ce900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.113+0000 7fdfe23ce900 20 started sync module instance, tier type = 2026-03-20T11:47:08.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.113+0000 7fdfe23ce900 20 generating connection object for zone b1 id efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c 2026-03-20T11:47:08.111 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.113+0000 7fdfe23ce900 20 started zone id=cd52b229-0d4a-43bf-b4ba-156aa19e7aef (name=b2) with tier type = 2026-03-20T11:47:08.122 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.124+0000 7fdfe23ce900 20 add_watcher() i=3 2026-03-20T11:47:08.122 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.124+0000 7fdfe23ce900 20 add_watcher() i=1 2026-03-20T11:47:08.122 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.124+0000 7fdfe23ce900 20 add_watcher() i=7 2026-03-20T11:47:08.122 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.124+0000 7fdfe23ce900 20 add_watcher() i=2 2026-03-20T11:47:08.122 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.124+0000 7fdfe23ce900 20 add_watcher() i=0 2026-03-20T11:47:08.123 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.125+0000 7fdfe23ce900 20 add_watcher() i=6 2026-03-20T11:47:08.123 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.125+0000 7fdfe23ce900 20 add_watcher() i=4 2026-03-20T11:47:08.123 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.125+0000 7fdfe23ce900 20 add_watcher() i=5 2026-03-20T11:47:08.123 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.125+0000 7fdfe23ce900 2 all 8 watchers are set, enabling cache 2026-03-20T11:47:08.126 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.128+0000 7fdfe23ce900 20 rgw_check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2026-03-20T11:47:08.126 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.128+0000 7fdfe23ce900 20 rgw_check_secure_mon_conn(): mode 1 is insecure 2026-03-20T11:47:08.126 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.128+0000 7fdfe23ce900 5 note: GC not initialized 2026-03-20T11:47:08.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.183+0000 7fdfe23ce900 20 init_complete bucket index max shards: 11 2026-03-20T11:47:08.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.183+0000 7fdfe23ce900 20 Filter name: none 2026-03-20T11:47:08.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.183+0000 7fdf8a7e4640 20 reqs_thread_entry: start 2026-03-20T11:47:08.181 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.183+0000 7fdf88fe1640 20 reqs_thread_entry: start 2026-03-20T11:47:08.182 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.184+0000 7fdfe23ce900 10 rgw bucket trim manager: Watching bilog.trim 2026-03-20T11:47:08.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.184+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39522330:12BucketTrimCR: operate() 2026-03-20T11:47:08.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.184+0000 7fdfe23ce900 10 fetching active bucket counters 2026-03-20T11:47:08.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.184+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39524a50:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:08.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.184+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39524a50:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:08.183 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.184+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is io blocked 2026-03-20T11:47:08.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.187+0000 7fdfb8ff9640 20 trim: get_bucket_counters: [] 2026-03-20T11:47:08.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.187+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39524a50:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:08.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.187+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39524a50:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:08.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.187+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39524a50:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:08.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.187+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39524a50:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:08.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.187+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39522330:12BucketTrimCR: operate() 2026-03-20T11:47:08.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.187+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:08.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.187+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:08.185 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.187+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is io blocked 2026-03-20T11:47:08.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.188+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:08.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.188+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:08.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.188+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:08.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.188+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:08.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.188+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39522330:12BucketTrimCR: operate() 2026-03-20T11:47:08.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.188+0000 7fdfe23ce900 10 listing cold buckets from marker=16:ca472e1e:root::.bucket.meta.dipepu-54%3a4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14:head 2026-03-20T11:47:08.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.188+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:14MetadataListCR: operate() 2026-03-20T11:47:08.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.188+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:14MetadataListCR: operate() 2026-03-20T11:47:08.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.188+0000 7fdfe23ce900 20 enqueued request req=0x7fdf98002a10 2026-03-20T11:47:08.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.188+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.188+0000 7fdfe23ce900 20 req: 0x7fdf98002a10 2026-03-20T11:47:08.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.188+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is io blocked 2026-03-20T11:47:08.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.188+0000 7fdfa4ff9640 20 dequeued request req=0x7fdf98002a10 2026-03-20T11:47:08.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.188+0000 7fdfa4ff9640 20 RGWWQ: empty 2026-03-20T11:47:08.186 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.188+0000 7fdfa4ff9640 20 rgw async rados processor: starting metadata listing at 16:ca472e1e:root::.bucket.meta.dipepu-54%3a4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14:head 2026-03-20T11:47:08.188 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.190+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:47:08.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.191+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-40 2026-03-20T11:47:08.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.191+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-22 2026-03-20T11:47:08.189 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.191+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:47:08.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.192+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-56 2026-03-20T11:47:08.190 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.192+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 2026-03-20T11:47:08.192 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.194+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:47:08.193 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.195+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:47:08.194 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.196+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:47:08.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.197+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-53 2026-03-20T11:47:08.195 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.197+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:47:08.196 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.198+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:47:08.197 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.199+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:47:08.198 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.200+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:47:08.199 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.201+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:47:08.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.202+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-34 2026-03-20T11:47:08.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.202+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-21 2026-03-20T11:47:08.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.202+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-14 2026-03-20T11:47:08.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.202+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-50 2026-03-20T11:47:08.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.202+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-57 2026-03-20T11:47:08.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.202+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:47:08.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.202+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-39 2026-03-20T11:47:08.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.202+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:47:08.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.202+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-37 2026-03-20T11:47:08.200 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.202+0000 7fdfa4ff9640 20 rgw async rados processor: restarting metadata listing 2026-03-20T11:47:08.208 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.210+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-10 2026-03-20T11:47:08.208 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.210+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:47:08.215 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.217+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:47:08.221 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.223+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:47:08.227 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.229+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:47:08.233 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.235+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:47:08.239 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.241+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:47:08.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.246+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-19 2026-03-20T11:47:08.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.246+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-3 2026-03-20T11:47:08.244 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.246+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:47:08.250 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.252+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:47:08.255 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.257+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-17 2026-03-20T11:47:08.255 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.257+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:47:08.260 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.262+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-38 2026-03-20T11:47:08.260 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.262+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:47:08.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.267+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-24 2026-03-20T11:47:08.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.267+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-36 2026-03-20T11:47:08.265 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.267+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:47:08.270 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.272+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:47:08.275 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.277+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:47:08.280 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.281+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-44 2026-03-20T11:47:08.280 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.281+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-35 2026-03-20T11:47:08.280 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.281+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:47:08.284 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.286+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-16 2026-03-20T11:47:08.284 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.286+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:47:08.289 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.291+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:47:08.293 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.295+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:47:08.298 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.300+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-55 2026-03-20T11:47:08.298 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.300+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-13 2026-03-20T11:47:08.298 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.300+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-48 2026-03-20T11:47:08.298 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.300+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-25 2026-03-20T11:47:08.298 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.300+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-51 2026-03-20T11:47:08.298 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.300+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:47:08.302 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.304+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:47:08.307 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.309+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-46 2026-03-20T11:47:08.307 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.309+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:47:08.311 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.313+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:47:08.315 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.317+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:47:08.319 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.321+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-26 2026-03-20T11:47:08.319 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.321+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-20 2026-03-20T11:47:08.319 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.321+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:47:08.323 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.325+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:47:08.327 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.329+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-1 2026-03-20T11:47:08.327 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.329+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:47:08.332 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.334+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:47:08.336 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.338+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:47:08.340 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.342+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-52 2026-03-20T11:47:08.340 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.342+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-32 2026-03-20T11:47:08.340 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.342+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-42 2026-03-20T11:47:08.340 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.342+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:47:08.343 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.345+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:47:08.347 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.349+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-7 2026-03-20T11:47:08.347 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.349+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-54 2026-03-20T11:47:08.347 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.349+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-49 2026-03-20T11:47:08.347 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.349+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-47 2026-03-20T11:47:08.347 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.349+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 2026-03-20T11:47:08.350 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.352+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:47:08.353 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.355+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:47:08.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.358+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-11 2026-03-20T11:47:08.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.358+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-12 2026-03-20T11:47:08.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.358+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-41 2026-03-20T11:47:08.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.358+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-23 2026-03-20T11:47:08.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.358+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-15 2026-03-20T11:47:08.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.358+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-45 2026-03-20T11:47:08.356 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.358+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:47:08.359 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.361+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:47:08.362 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.364+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-9 2026-03-20T11:47:08.362 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.364+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:47:08.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.366+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-29 2026-03-20T11:47:08.364 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.366+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-31 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfa4ff9640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:14MetadataListCR: operate() 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:14MetadataListCR: operate() 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:14MetadataListCR: operate() 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:14MetadataListCR: operate() 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39522330:12BucketTrimCR: operate() 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 4 collected 50 buckets for trim 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39523a40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 cr:s=0x558b3956d1b0:op=0x558b395272b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39528580:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b39529850:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b3952ab20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 cr:s=0x558b3956d1b0:op=0x558b3952b270:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3952b9c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3952c110:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b3952ab20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 enqueued request req=0x558b3952c860 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 req: 0x558b3952c860 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is io blocked 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 cr:s=0x558b3956d1b0:op=0x558b3952b270:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 enqueued request req=0x558b3952cc00 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 req: 0x558b3952c860 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 req: 0x558b3952cc00 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 run: stack=0x558b3956d1b0 is io blocked 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3952b9c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 enqueued request req=0x558b3952d020 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 req: 0x558b3952c860 2026-03-20T11:47:08.368 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 req: 0x558b3952cc00 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 req: 0x558b3952d020 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is io blocked 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3952c110:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 enqueued request req=0x558b3952d440 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 req: 0x558b3952c860 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 req: 0x558b3952cc00 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 req: 0x558b3952d020 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 req: 0x558b3952d440 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdfe23ce900 20 run: stack=0x558b39526660 is io blocked 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf9f7fe640 20 dequeued request req=0x558b3952c860 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf9f7fe640 20 RGWWQ: 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf9f7fe640 20 req: 0x558b3952cc00 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf9f7fe640 20 req: 0x558b3952d020 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf9f7fe640 20 req: 0x558b3952d440 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf9f7fe640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 : miss 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf9f7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf8c7e8640 20 dequeued request req=0x558b3952cc00 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf8c7e8640 20 RGWWQ: 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf8c7e8640 20 req: 0x558b3952d020 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf8c7e8640 20 req: 0x558b3952d440 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf8c7e8640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 : miss 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf8c7e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf9e7fc640 20 dequeued request req=0x558b3952d020 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf9e7fc640 20 RGWWQ: 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf9e7fc640 20 req: 0x558b3952d440 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf9ffff640 20 dequeued request req=0x558b3952d440 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf9ffff640 20 RGWWQ: empty 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf9ffff640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 : miss 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf9ffff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf9e7fc640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 : miss 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf8c7e8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf9e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf8c7e8640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 info.flags=0x17 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf8c7e8640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 to cache LRU end 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf8c7e8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf8c7e8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf8c7e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf8c7e8640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-54 : miss 2026-03-20T11:47:08.369 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.369+0000 7fdf8c7e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.371 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9ffff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=386 2026-03-20T11:47:08.371 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9ffff640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 info.flags=0x17 2026-03-20T11:47:08.371 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9ffff640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 to cache LRU end 2026-03-20T11:47:08.371 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9ffff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.371 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9ffff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.371 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9ffff640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 2026-03-20T11:47:08.371 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9ffff640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-5 : miss 2026-03-20T11:47:08.371 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9ffff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.371 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9e7fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.371 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9e7fc640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 info.flags=0x17 2026-03-20T11:47:08.371 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9e7fc640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 to cache LRU end 2026-03-20T11:47:08.371 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9e7fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.371 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9e7fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.371 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9e7fc640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9e7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9e7fc640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-32 : miss 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9f7fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=387 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9f7fe640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 info.flags=0x17 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9f7fe640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 to cache LRU end 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9f7fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9f7fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9f7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9f7fe640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-61 : miss 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9f7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf8c7e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf8c7e8640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-54 info.flags=0x0 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf8c7e8640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-54 to cache LRU end 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf8c7e8640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-54 : miss 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf8c7e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9ffff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9ffff640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-5 info.flags=0x0 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9ffff640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-5 to cache LRU end 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9ffff640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-5 : miss 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9ffff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9e7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9e7fc640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-32 info.flags=0x0 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9e7fc640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-32 to cache LRU end 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9f7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9f7fe640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-61 info.flags=0x0 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9f7fe640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-61 to cache LRU end 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9f7fe640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-61 : miss 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9f7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9e7fc640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-32 : miss 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf9e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf8c7e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf8c7e8640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-54 info.flags=0x0 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf8c7e8640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-54 to cache LRU end 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf8c7e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14): adding source pipe: {s={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf8c7e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14): adding dest pipe: {s={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf8c7e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf8c7e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdf8c7e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdfe23ce900 20 cr:s=0x558b3956d1b0:op=0x558b3952b270:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdfe23ce900 20 cr:s=0x558b3956d1b0:op=0x558b3952b270:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdfe23ce900 20 cr:s=0x558b3956d1b0:op=0x558b3952b270:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdfe23ce900 20 cr:s=0x558b3956d1b0:op=0x558b3952b270:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdfe23ce900 20 cr:s=0x558b3956d1b0:op=0x558b395272b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3952d860:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdfe23ce900 20 cr:s=0x558b3956d1b0:op=0x558b395272b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdfe23ce900 20 run: stack=0x558b3956d1b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3952b270:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.370+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3952b270:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.372 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdf9e7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdf9e7fc640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-32 info.flags=0x0 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdf9e7fc640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-32 to cache LRU end 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdf9ffff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdf9ffff640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-5 info.flags=0x0 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdf9ffff640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-5 to cache LRU end 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdf9ffff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10): adding source pipe: {s={b=dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdf9ffff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10): adding dest pipe: {s={b=dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdf9ffff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdf9ffff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdf9e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37): adding source pipe: {s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdf9ffff640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdf9e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37): adding dest pipe: {s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 20111c0f0074a2171e789d99911219205b1d21ed69dedfc309ae44deab08ab56 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:20111c0f0074a2171e789d99911219205b1d21ed69dedfc309ae44deab08ab56 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdf9f7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdf9f7fe640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-61 info.flags=0x0 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdf9f7fe640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-61 to cache LRU end 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdf9f7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21): adding source pipe: {s={b=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdf9f7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21): adding dest pipe: {s={b=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = d2b56f9cb0222d50d14ab6df8db8713435bea82014c317895aa01959be9121dd 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdf9f7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdf9f7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=d2b56f9cb0222d50d14ab6df8db8713435bea82014c317895aa01959be9121dd 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdf9f7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=0, curl_handle=0x558b3952f650 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 run: stack=0x558b39357a70 is io blocked 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3952c110:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b3952ab20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3952c110:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b3952ab20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.373 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3952c110:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b3952ab20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3952c110:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b3952ab20:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b39529850:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39523a40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdf88fe1640 20 link_request req_data=0x558b39530150 req_data->id=0, curl_handle=0x558b3952f650 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 cr:s=0x558b3952afd0:op=0x558b3952c110:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b39529850:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 run: stack=0x558b39526660 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 cr:s=0x558b3952ad30:op=0x558b39532080:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39523a40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 cr:s=0x558b3952afd0:op=0x558b395326a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 cr:s=0x558b3952ad30:op=0x558b39532cd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 cr:s=0x558b3952afd0:op=0x558b395326a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 457991b4c6b02e3ed17079576dd809106d9dfb4af0c6253d3d7fedc63d9ce4c2 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:457991b4c6b02e3ed17079576dd809106d9dfb4af0c6253d3d7fedc63d9ce4c2 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = be757e008cf7ee375fd61e800b7613a48f8e1e19b45b12e79d74a6d6f50c9238 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=be757e008cf7ee375fd61e800b7613a48f8e1e19b45b12e79d74a6d6f50c9238 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-5%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=1, curl_handle=0x558b39534860 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 run: stack=0x558b3952afd0 is io blocked 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 cr:s=0x558b3952ad30:op=0x558b39532cd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.374 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = cedbc0fb027f3c9cb1b560bb9a6df789d994fc2dbf44f6c4bba822b387d2b519 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:cedbc0fb027f3c9cb1b560bb9a6df789d994fc2dbf44f6c4bba822b387d2b519 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 75d42563a83afddffe85ce9340ab745f050fa529e3aae92317ce6c876fa978f9 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=75d42563a83afddffe85ce9340ab745f050fa529e3aae92317ce6c876fa978f9 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-61%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdf9e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=2, curl_handle=0x558b39537db0 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 run: stack=0x558b3952ad30 is io blocked 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdf9e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdf88fe1640 20 link_request req_data=0x558b395351a0 req_data->id=1, curl_handle=0x558b39534860 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdf88fe1640 20 link_request req_data=0x558b39538500 req_data->id=2, curl_handle=0x558b39537db0 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdf9e7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3952b9c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3952b9c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3952b9c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3952b9c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39528580:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b39539ed0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39528580:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3953a4f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3953a4f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.375 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = d1aee89dcfbe421df9685165d847b5b9ebee5000ca2171923c5cb1ba34b2454e 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:d1aee89dcfbe421df9685165d847b5b9ebee5000ca2171923c5cb1ba34b2454e 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = e70a6a1fef9918ea8d3f1760f2ab9636ba08b1f85708deba8411f93e1e696945 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=e70a6a1fef9918ea8d3f1760f2ab9636ba08b1f85708deba8411f93e1e696945 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=3, curl_handle=0x558b3953c0d0 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is io blocked 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.371+0000 7fdf88fe1640 20 link_request req_data=0x558b3953c7e0 req_data->id=3, curl_handle=0x558b3953c0d0 2026-03-20T11:47:08.376 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b3952afd0:op=0x558b395326a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b3952afd0:op=0x558b395326a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b3952afd0:op=0x558b3952c110:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 15 stack 0x558b3952afd0 end 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 run: stack=0x558b3952afd0 is done 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b3952ad30:op=0x558b39532cd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b3952ad30:op=0x558b39532cd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b39529850:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 collect(): s=0x558b39526660 stack=0x558b3952afd0 is complete 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b3952ad30:op=0x558b39532080:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 15 stack 0x558b3952ad30 end 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 run: stack=0x558b3952ad30 is done 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3952c110:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.0 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.1 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.2 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.3 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.4 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.5 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.6 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.7 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.8 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.9 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10.10 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 run: stack=0x558b39526660 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39523a40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 collect(): s=0x558b3943dd50 stack=0x558b3952ad30 is complete 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b3952afd0:op=0x558b39536920:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b39534bf0:op=0x558b39532080:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b39537800:op=0x558b39532670:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b39533400:op=0x558b39532e10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b395338e0:op=0x558b3953e170:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b3953f300:op=0x558b3953e910:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b3953ffe0:op=0x558b3953f5f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b395402d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b395419c0:op=0x558b39540fd0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b395426e0:op=0x558b39541cf0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b39543400:op=0x558b39542a10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39543690:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.0 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.1 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.2 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.3 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.4 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.5 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.6 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.7 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.8 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.9 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21.10 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.377 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b3952afd0:op=0x558b39536920:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.377 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.375+0000 7f767dbed640 -1 req 13088447774104489105 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.379 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 run: stack=0x558b3952afd0 is io blocked 2026-03-20T11:47:08.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b39534bf0:op=0x558b39532080:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 run: stack=0x558b39534bf0 is io blocked 2026-03-20T11:47:08.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b39537800:op=0x558b39532670:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 run: stack=0x558b39537800 is io blocked 2026-03-20T11:47:08.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b39533400:op=0x558b39532e10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 run: stack=0x558b39533400 is io blocked 2026-03-20T11:47:08.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b395338e0:op=0x558b3953e170:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 run: stack=0x558b395338e0 is io blocked 2026-03-20T11:47:08.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b3953f300:op=0x558b3953e910:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 run: stack=0x558b3953f300 is io blocked 2026-03-20T11:47:08.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b3953ffe0:op=0x558b3953f5f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 run: stack=0x558b3953ffe0 is io blocked 2026-03-20T11:47:08.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b395402d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 run: stack=0x558b39540cc0 is io blocked 2026-03-20T11:47:08.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b395419c0:op=0x558b39540fd0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 run: stack=0x558b395419c0 is io blocked 2026-03-20T11:47:08.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b395426e0:op=0x558b39541cf0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 run: stack=0x558b395426e0 is io blocked 2026-03-20T11:47:08.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b39543400:op=0x558b39542a10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 run: stack=0x558b39543400 is io blocked 2026-03-20T11:47:08.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b3952ad30:op=0x558b39544060:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b39545640:op=0x558b39544c50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.380 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b39546370:op=0x558b39545980:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b395470b0:op=0x558b395466c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b390dbc30:op=0x558b390db640:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b390dc930:op=0x558b390dbf40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b390dd660:op=0x558b390dcc70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b390de390:op=0x558b390dd9a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b390df0a0:op=0x558b390de6b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b395798b0:op=0x558b390df490:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b3957a550:op=0x558b39579b60:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.374+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3952b270:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 cr:s=0x558b3952ad30:op=0x558b39544060:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 run: stack=0x558b3952ad30 is io blocked 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 cr:s=0x558b39545640:op=0x558b39544c50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 run: stack=0x558b39545640 is io blocked 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 cr:s=0x558b39546370:op=0x558b39545980:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 run: stack=0x558b39546370 is io blocked 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 cr:s=0x558b395470b0:op=0x558b395466c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 run: stack=0x558b395470b0 is io blocked 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 cr:s=0x558b390dbc30:op=0x558b390db640:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 run: stack=0x558b390dbc30 is io blocked 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 cr:s=0x558b390dc930:op=0x558b390dbf40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 run: stack=0x558b390dc930 is io blocked 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 cr:s=0x558b390dd660:op=0x558b390dcc70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 run: stack=0x558b390dd660 is io blocked 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 cr:s=0x558b390de390:op=0x558b390dd9a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 run: stack=0x558b390de390 is io blocked 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 cr:s=0x558b390df0a0:op=0x558b390de6b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 run: stack=0x558b390df0a0 is io blocked 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 cr:s=0x558b395798b0:op=0x558b390df490:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 run: stack=0x558b395798b0 is io blocked 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 cr:s=0x558b3957a550:op=0x558b39579b60:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 run: stack=0x558b3957a550 is io blocked 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3952b270:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3953a4f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3953a4f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3952b270:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b39539ed0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 15 stack 0x558b3952bf80 end 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is done 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3952b270:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39528580:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 collect(): s=0x558b3943e140 stack=0x558b3952bf80 is complete 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39528580:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 15 stack 0x558b3943e140 end 2026-03-20T11:47:08.381 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is done 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3952d860:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 15 stack 0x558b39357a70 end 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 run: stack=0x558b39357a70 is done 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.376+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 cr:s=0x558b3956d1b0:op=0x558b395272b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 collect(): s=0x558b3956d1b0 stack=0x558b39357a70 is complete 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 10 trimming bilogs for bucket=:dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) markers=[00000000002.32.1,00000000002.28.1,00000000002.50.1,00000000002.28.1,00000000002.28.1,00000000002.47.1,00000000002.33.1,00000000002.30.1,00000000002.39.1,00000000002.57.1,00000000002.22.1], shards=11 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39528580:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 cr:s=0x558b3956d1b0:op=0x558b3952d860:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 10 trimming bilog shard 0 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.32.1 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 10 trimming bilog shard 1 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.28.1 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 10 trimming bilog shard 2 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.50.1 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 10 trimming bilog shard 3 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.28.1 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 10 trimming bilog shard 4 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.28.1 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 10 trimming bilog shard 5 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.47.1 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 10 trimming bilog shard 6 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.33.1 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 10 trimming bilog shard 7 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.30.1 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 10 trimming bilog shard 8 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.39.1 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 10 trimming bilog shard 9 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.57.1 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 10 trimming bilog shard 10 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.22.1 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 run: stack=0x558b3956d1b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39596f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b395a0800:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395a1000:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 cr:s=0x558b3953c460:op=0x558b39539eb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 cr:s=0x558b3957dcd0:op=0x558b3953a6b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 cr:s=0x558b3957d090:op=0x558b39594fe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 cr:s=0x558b395a1a80:op=0x558b395957e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 cr:s=0x558b39534fc0:op=0x558b390ca490:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 cr:s=0x558b3953dfb0:op=0x558b390cac90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 cr:s=0x558b3953c600:op=0x558b3959eea0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 cr:s=0x558b39598bf0:op=0x558b3959f6a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 cr:s=0x558b3952ff70:op=0x558b3959fea0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39596f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 enqueued request req=0x558b390d20e0 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 req: 0x558b390d20e0 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is io blocked 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b395a0800:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.0 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 run: stack=0x558b39357a70 is io blocked 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395a1000:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.1 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is io blocked 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 cr:s=0x558b3953c460:op=0x558b39539eb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.2 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdf9dffb640 20 dequeued request req=0x558b390d20e0 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdf9dffb640 20 RGWWQ: empty 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 run: stack=0x558b3953c460 is io blocked 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 cr:s=0x558b3957dcd0:op=0x558b3953a6b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.3 2026-03-20T11:47:08.382 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 run: stack=0x558b3957dcd0 is io blocked 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 cr:s=0x558b3957d090:op=0x558b39594fe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.4 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdf9dffb640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 : miss 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdf9dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 run: stack=0x558b3957d090 is io blocked 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 cr:s=0x558b395a1a80:op=0x558b395957e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.5 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 run: stack=0x558b395a1a80 is io blocked 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 cr:s=0x558b39534fc0:op=0x558b390ca490:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.6 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdf9dffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdf9dffb640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 info.flags=0x17 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdf9dffb640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 to cache LRU end 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdf9dffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdf9dffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdf9dffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdf9dffb640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-19 : miss 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 run: stack=0x558b39534fc0 is io blocked 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 cr:s=0x558b3953dfb0:op=0x558b390cac90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.7 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdf9dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 run: stack=0x558b3953dfb0 is io blocked 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 cr:s=0x558b3953c600:op=0x558b3959eea0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.8 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 run: stack=0x558b3953c600 is io blocked 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 cr:s=0x558b39598bf0:op=0x558b3959f6a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.9 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 run: stack=0x558b39598bf0 is io blocked 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.377+0000 7fdfe23ce900 20 cr:s=0x558b3952ff70:op=0x558b3959fea0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.378+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.10 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.378+0000 7fdfe23ce900 20 run: stack=0x558b3952ff70 is io blocked 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.378+0000 7fdf9dffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.378+0000 7fdf9dffb640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-19 info.flags=0x0 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.378+0000 7fdf9dffb640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-19 to cache LRU end 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.378+0000 7fdf9dffb640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-19 : miss 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.378+0000 7fdf9dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdf9dffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdf9dffb640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-19 info.flags=0x0 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdf9dffb640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-19 to cache LRU end 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdf9dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24): adding source pipe: {s={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdf9dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24): adding dest pipe: {s={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdf9dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdf9dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdf9dffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39596f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39596f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39596f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39596f10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39528580:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b395b5470:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39528580:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596ee0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596ee0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.383 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 3f0fe821d20d639d2de413541f1e5041b10ae661eef11c84d9621be214882967 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:3f0fe821d20d639d2de413541f1e5041b10ae661eef11c84d9621be214882967 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = b3e7df057a67aaec6252d8fa1d5c61a47b71500a2ab6f843e51e4c3ed8e3dba9 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b3e7df057a67aaec6252d8fa1d5c61a47b71500a2ab6f843e51e4c3ed8e3dba9 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=4, curl_handle=0x558b3953c0d0 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdfe23ce900 20 run: stack=0x558b3957acd0 is io blocked 2026-03-20T11:47:08.384 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.380+0000 7fdf88fe1640 20 link_request req_data=0x558b3953c7e0 req_data->id=4, curl_handle=0x558b3953c0d0 2026-03-20T11:47:08.384 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.382+0000 7f7670bd3640 -1 req 4530315349342897823 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.386 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.383+0000 7fdfe23ce900 20 cr:s=0x558b39533400:op=0x558b39532e10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.386 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.383+0000 7fdfe23ce900 20 cr:s=0x558b39533400:op=0x558b39532e10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.386 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.383+0000 7fdfe23ce900 20 cr:s=0x558b395338e0:op=0x558b3953e170:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.386 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.383+0000 7fdfe23ce900 20 cr:s=0x558b39533400:op=0x558b39532e10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.386 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.383+0000 7fdfe23ce900 20 cr:s=0x558b395338e0:op=0x558b3953e170:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.386 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.383+0000 7fdfe23ce900 20 cr:s=0x558b39533400:op=0x558b39532e10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.386 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.383+0000 7fdfe23ce900 15 stack 0x558b39533400 end 2026-03-20T11:47:08.386 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.383+0000 7fdfe23ce900 20 run: stack=0x558b39533400 is done 2026-03-20T11:47:08.386 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.383+0000 7fdfe23ce900 20 cr:s=0x558b395338e0:op=0x558b3953e170:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.386 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.383+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3952c110:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:08.386 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.383+0000 7fdfe23ce900 20 run: stack=0x558b39526660 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.386 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.383+0000 7fdfe23ce900 20 cr:s=0x558b390dbc30:op=0x558b390db640:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.386 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.383+0000 7fdfe23ce900 20 cr:s=0x558b395338e0:op=0x558b3953e170:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.386 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.383+0000 7fdfe23ce900 15 stack 0x558b395338e0 end 2026-03-20T11:47:08.386 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.383+0000 7fdfe23ce900 20 run: stack=0x558b395338e0 is done 2026-03-20T11:47:08.386 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.383+0000 7fdfe23ce900 20 cr:s=0x558b390dbc30:op=0x558b390db640:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.386 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.383+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3952c110:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:08.386 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.383+0000 7fdfe23ce900 20 run: stack=0x558b39526660 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.386 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.383+0000 7fdfe23ce900 20 cr:s=0x558b390dbc30:op=0x558b390db640:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.386 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.383+0000 7fdfe23ce900 20 cr:s=0x558b390dbc30:op=0x558b390db640:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.383+0000 7fdfe23ce900 15 stack 0x558b390dbc30 end 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.383+0000 7fdfe23ce900 20 run: stack=0x558b390dbc30 is done 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.383+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39543690:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.383+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b395402d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b395402d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 20 cr:s=0x558b3953ffe0:op=0x558b3953f5f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 20 cr:s=0x558b395426e0:op=0x558b39541cf0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 20 cr:s=0x558b3953f300:op=0x558b3953e910:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 20 cr:s=0x558b395419c0:op=0x558b39540fd0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 20 cr:s=0x558b395798b0:op=0x558b390df490:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 20 cr:s=0x558b39598bf0:op=0x558b3959f6a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 20 cr:s=0x558b39598bf0:op=0x558b3959f6a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 15 stack 0x558b39598bf0 end 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 20 stack->operate() returned ret=-61 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 20 run: stack=0x558b39598bf0 is done 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b395402d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 20 cr:s=0x558b3953ffe0:op=0x558b3953f5f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 20 cr:s=0x558b395426e0:op=0x558b39541cf0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 20 cr:s=0x558b3953f300:op=0x558b3953e910:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 20 cr:s=0x558b395419c0:op=0x558b39540fd0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 20 cr:s=0x558b395798b0:op=0x558b390df490:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 20 cr:s=0x558b3956d1b0:op=0x558b3952d860:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 20 run: stack=0x558b3956d1b0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 20 cr:s=0x558b39546370:op=0x558b39545980:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 20 cr:s=0x558b39537800:op=0x558b39532670:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b395a0800:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b395a0800:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 15 stack 0x558b39357a70 end 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 20 stack->operate() returned ret=-61 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 20 run: stack=0x558b39357a70 is done 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 20 cr:s=0x558b390df0a0:op=0x558b390de6b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.384+0000 7fdfe23ce900 20 cr:s=0x558b390dd660:op=0x558b390dcc70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39545640:op=0x558b39544c50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3953dfb0:op=0x558b390cac90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3953dfb0:op=0x558b390cac90:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b3953dfb0 end 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 stack->operate() returned ret=-61 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b3953dfb0 is done 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3953c460:op=0x558b39539eb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3953c460:op=0x558b39539eb0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b3953c460 end 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 stack->operate() returned ret=-61 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b3953c460 is done 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3952ff70:op=0x558b3959fea0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3952ff70:op=0x558b3959fea0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b3952ff70 end 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 stack->operate() returned ret=-61 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b3952ff70 is done 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b390dc930:op=0x558b390dbf40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b395402d0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b39540cc0 end 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b39540cc0 is done 2026-03-20T11:47:08.387 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3953ffe0:op=0x558b3953f5f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b395426e0:op=0x558b39541cf0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3953f300:op=0x558b3953e910:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b395419c0:op=0x558b39540fd0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b395798b0:op=0x558b390df490:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39546370:op=0x558b39545980:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39537800:op=0x558b39532670:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3956d1b0:op=0x558b3952d860:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39534fc0:op=0x558b390ca490:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39534fc0:op=0x558b390ca490:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b39534fc0 end 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 stack->operate() returned ret=-61 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b39534fc0 is done 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b390df0a0:op=0x558b390de6b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b390dd660:op=0x558b390dcc70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3952afd0:op=0x558b39536920:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b395470b0:op=0x558b395466c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395a1000:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395a1000:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b3952bf80 end 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 stack->operate() returned ret=-61 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is done 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39543400:op=0x558b39542a10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3957dcd0:op=0x558b3953a6b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3957dcd0:op=0x558b3953a6b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b3957dcd0 end 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 stack->operate() returned ret=-61 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b3957dcd0 is done 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3953c600:op=0x558b3959eea0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3953c600:op=0x558b3959eea0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b3953c600 end 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 stack->operate() returned ret=-61 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b3953c600 is done 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3957d090:op=0x558b39594fe0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3957d090:op=0x558b39594fe0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b3957d090 end 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 stack->operate() returned ret=-61 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b3957d090 is done 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3952ad30:op=0x558b39544060:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39534bf0:op=0x558b39532080:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b395a1a80:op=0x558b395957e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b395a1a80:op=0x558b395957e0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b395a1a80 end 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 stack->operate() returned ret=-61 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b395a1a80 is done 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39545640:op=0x558b39544c50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b390de390:op=0x558b390dd9a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.388 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3957a550:op=0x558b39579b60:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b390dc930:op=0x558b390dbf40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3952c110:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b39526660 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3953ffe0:op=0x558b3953f5f0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b3953ffe0 end 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b3953ffe0 is done 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b395426e0:op=0x558b39541cf0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b395426e0 end 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b395426e0 is done 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3953f300:op=0x558b3953e910:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b3953f300 end 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b3953f300 is done 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b395419c0:op=0x558b39540fd0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b395419c0 end 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b395419c0 is done 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b395798b0:op=0x558b390df490:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b395798b0 end 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b395798b0 is done 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39546370:op=0x558b39545980:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39537800:op=0x558b39532670:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3956d1b0:op=0x558b3952d860:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b390df0a0:op=0x558b390de6b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b390dd660:op=0x558b390dcc70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3952afd0:op=0x558b39536920:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b395470b0:op=0x558b395466c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39543400:op=0x558b39542a10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3952ad30:op=0x558b39544060:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39534bf0:op=0x558b39532080:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39545640:op=0x558b39544c50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b390de390:op=0x558b390dd9a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3957a550:op=0x558b39579b60:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b390dc930:op=0x558b390dbf40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3952c110:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39543690:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39546370:op=0x558b39545980:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b39546370 end 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b39546370 is done 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39537800:op=0x558b39532670:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b39537800 end 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b39537800 is done 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3956d1b0:op=0x558b3952d860:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b390df0a0:op=0x558b390de6b0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b390df0a0 end 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b390df0a0 is done 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b390dd660:op=0x558b390dcc70:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b390dd660 end 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b390dd660 is done 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3952afd0:op=0x558b39536920:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b395470b0:op=0x558b395466c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39543400:op=0x558b39542a10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3952ad30:op=0x558b39544060:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39534bf0:op=0x558b39532080:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39545640:op=0x558b39544c50:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b39545640 end 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b39545640 is done 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b390de390:op=0x558b390dd9a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.389 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3957a550:op=0x558b39579b60:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b390dc930:op=0x558b390dbf40:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b390dc930 end 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b390dc930 is done 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3952c110:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39543690:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3956d1b0:op=0x558b3952d860:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3952afd0:op=0x558b39536920:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b3952afd0 end 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b3952afd0 is done 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b395470b0:op=0x558b395466c0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b395470b0 end 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b395470b0 is done 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39543400:op=0x558b39542a10:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b39543400 end 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b39543400 is done 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3952ad30:op=0x558b39544060:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b3952ad30 end 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b3952ad30 is done 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39534bf0:op=0x558b39532080:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b39534bf0 end 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b39534bf0 is done 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b390de390:op=0x558b390dd9a0:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b390de390 end 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b390de390 is done 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3957a550:op=0x558b39579b60:19RGWRadosRemoveOidCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b3957a550 end 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b3957a550 is done 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3952c110:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39543690:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3956d1b0:op=0x558b3952d860:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3952c110:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39543690:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3956d1b0:op=0x558b3952d860:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3952c110:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39543690:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3956d1b0:op=0x558b3952d860:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3952c110:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39543690:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3956d1b0:op=0x558b3952d860:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3952c110:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39543690:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3956d1b0:op=0x558b3952d860:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3952c110:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39543690:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3956d1b0:op=0x558b3952d860:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b39529850:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39543690:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3956d1b0:op=0x558b395272b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 trim: trimmed bucket instance dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 15 stack 0x558b3956d1b0 end 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b3956d1b0 is done 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b390db640:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39543690:25BucketCleanIndexCollectCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b390db640:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 enqueued request req=0x558b390df1e0 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.390 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 req: 0x558b390df1e0 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b39526660 is io blocked 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39523a40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395272b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39532e10:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3959e140:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39532e10:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdf9d7fa640 20 dequeued request req=0x558b390df1e0 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdf9d7fa640 20 RGWWQ: empty 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 enqueued request req=0x558b395808a0 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 req: 0x558b395808a0 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is io blocked 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3959e140:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 enqueued request req=0x558b395473f0 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 req: 0x558b395808a0 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 req: 0x558b395473f0 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is io blocked 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdf9cff9640 20 dequeued request req=0x558b395808a0 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdf9cff9640 20 RGWWQ: 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdf9cff9640 20 req: 0x558b395473f0 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdf97fff640 20 dequeued request req=0x558b395473f0 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdf97fff640 20 RGWWQ: empty 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdf97fff640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 : miss 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.385+0000 7fdf97fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.386+0000 7fdf97fff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.386+0000 7fdf97fff640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 info.flags=0x17 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.386+0000 7fdf97fff640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 to cache LRU end 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.386+0000 7fdf97fff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.386+0000 7fdf97fff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.386+0000 7fdf97fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.386+0000 7fdf97fff640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-4 : miss 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.386+0000 7fdf97fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.386+0000 7fdf97fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.386+0000 7fdf97fff640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-4 info.flags=0x0 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.386+0000 7fdf97fff640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-4 to cache LRU end 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.386+0000 7fdf97fff640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-4 : miss 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.386+0000 7fdf97fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdf97fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdf97fff640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-4 info.flags=0x0 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdf97fff640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-4 to cache LRU end 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdf97fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5): adding source pipe: {s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdf97fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5): adding dest pipe: {s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdf97fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdf97fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdf97fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3959e140:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3959e140:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3959e140:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3959e140:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395272b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3952b1e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395272b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957c750:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957c750:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.391 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = f14cf820c920e0a436552c5808fe3abf7bd62318f05d220b008c54b31aa93603 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:f14cf820c920e0a436552c5808fe3abf7bd62318f05d220b008c54b31aa93603 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 602544cc213c204de5e08c7f32fa0d2ccf3d72f98a7ce3ccb0c797db9d16b41d 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=602544cc213c204de5e08c7f32fa0d2ccf3d72f98a7ce3ccb0c797db9d16b41d 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=5, curl_handle=0x558b3952f650 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdfe23ce900 20 run: stack=0x558b39540cc0 is io blocked 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.387+0000 7fdf88fe1640 20 link_request req_data=0x558b39530150 req_data->id=5, curl_handle=0x558b3952f650 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.388+0000 7fdf9cff9640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 info.flags=0x11 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.388+0000 7fdf9cff9640 10 rgw async rados processor: moving b2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 to cache LRU end 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.388+0000 7fdf9cff9640 10 rgw async rados processor: distributing notification oid=b2.rgw.control:notify.6 cni=[op: 0, obj: b2.rgw.meta:root:.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21, ofs0, ns] 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.388+0000 7fdf9d7fa640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 info.flags=0x11 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.388+0000 7fdf9d7fa640 10 rgw async rados processor: moving b2.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 to cache LRU end 2026-03-20T11:47:08.392 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.388+0000 7fdf9d7fa640 10 rgw async rados processor: distributing notification oid=b2.rgw.control:notify.5 cni=[op: 0, obj: b2.rgw.meta:root:.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10, ofs0, ns] 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.388+0000 7fdfb8ff9640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 257698037874 cookie 94056450814576 notifier 4747 bl.length()=565 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.388+0000 7fdfb8ff9640 10 rgw watcher librados: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 info.flags=0x11 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.388+0000 7fdfb8ff9640 10 rgw watcher librados: moving b2.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 to cache LRU end 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.388+0000 7fdfb8ff9640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 257698038017 cookie 94056450801040 notifier 4747 bl.length()=567 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.388+0000 7fdfb8ff9640 10 rgw watcher librados: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 info.flags=0x11 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.388+0000 7fdfb8ff9640 10 rgw watcher librados: moving b2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 to cache LRU end 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdf9cff9640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-61[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21]): orig_sources= new_sources= 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdf9cff9640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-61[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21]): potential sources added=[] removed=[] 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdf9cff9640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-61[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21]): orig_dests= new_dests= 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdf9cff9640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-61[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21]): potential dests added=[] removed=[] 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39532e10:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39532e10:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39532e10:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39532e10:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39523a40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b3957ea30:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b3957ea30:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdf9d7fa640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-5[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10]): orig_sources= new_sources= 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdf9d7fa640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-5[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10]): potential sources added=[] removed=[] 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdfe23ce900 20 enqueued request req=0x558b39067f40 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdf9d7fa640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-5[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10]): orig_dests= new_dests= 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdfe23ce900 20 req: 0x558b39067f40 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdf9d7fa640 20 rgw async rados processor: handle_bi_update(): bucket=:dipepu-5[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10]): potential dests added=[] removed=[] 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is io blocked 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdf9d7fa640 20 dequeued request req=0x558b39067f40 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdf9d7fa640 20 RGWWQ: empty 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b390db640:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b390db640:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b390db640:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b390db640:26RGWPutBucketInstanceInfoCR: operate() 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b39529850:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3959e140:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3959e140:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdfe23ce900 20 enqueued request req=0x558b39061af0 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdfe23ce900 20 req: 0x558b39061af0 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdfe23ce900 20 run: stack=0x558b39526660 is io blocked 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdf96ffd640 20 dequeued request req=0x558b39061af0 2026-03-20T11:47:08.393 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.389+0000 7fdf96ffd640 20 RGWWQ: empty 2026-03-20T11:47:08.393 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.390+0000 7f7652396640 -1 req 16072742596668129350 0.003000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.394 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.390+0000 7fdf96ffd640 10 rgw async rados processor: removing b2.rgw.meta+root+.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 from cache 2026-03-20T11:47:08.394 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.390+0000 7fdf96ffd640 10 rgw async rados processor: distributing notification oid=b2.rgw.control:notify.5 cni=[op: 1, obj: b2.rgw.meta:root:.bucket.meta.dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10, ofs0, ns] 2026-03-20T11:47:08.394 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.390+0000 7fdf9d7fa640 10 rgw async rados processor: removing b2.rgw.meta+root+.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 from cache 2026-03-20T11:47:08.394 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.390+0000 7fdf9d7fa640 10 rgw async rados processor: distributing notification oid=b2.rgw.control:notify.6 cni=[op: 1, obj: b2.rgw.meta:root:.bucket.meta.dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21, ofs0, ns] 2026-03-20T11:47:08.394 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfa7fff640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 257698038018 cookie 94056450801040 notifier 4747 bl.length()=206 2026-03-20T11:47:08.394 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfb8ff9640 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 257698037875 cookie 94056450814576 notifier 4747 bl.length()=205 2026-03-20T11:47:08.394 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3959e140:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:47:08.394 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3959e140:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b3957ea30:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3959e140:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b3957ea30:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3959e140:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b3957ea30:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b39529850:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 trim: trimmed bucket instance dipepu-5:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.10 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 15 stack 0x558b39526660 end 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 run: stack=0x558b39526660 is done 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b3957ea30:29RGWRemoveBucketInstanceInfoCR: operate() 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39523a40:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 trim: trimmed bucket instance dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 15 stack 0x558b3943dd50 end 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is done 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b39529850:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3959e140:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3959e140:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 enqueued request req=0x558b395336f0 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 req: 0x558b395336f0 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 run: stack=0x558b39526660 is io blocked 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b395996f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdf967fc640 20 dequeued request req=0x558b395336f0 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdf967fc640 20 RGWWQ: empty 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b395996f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdf967fc640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 : miss 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdf967fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 enqueued request req=0x558b39534d90 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 req: 0x558b39534d90 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is io blocked 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdf95ffb640 20 dequeued request req=0x558b39534d90 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdf95ffb640 20 RGWWQ: empty 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdf95ffb640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 : miss 2026-03-20T11:47:08.395 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdf95ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdf967fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdf967fc640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 info.flags=0x17 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdf967fc640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 to cache LRU end 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdf967fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdf967fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdf967fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdf967fc640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-53 : miss 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.391+0000 7fdf967fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf967fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf967fc640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-53 info.flags=0x0 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf967fc640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-53 to cache LRU end 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf967fc640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-53 : miss 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf967fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf95ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf95ffb640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 info.flags=0x17 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf95ffb640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 to cache LRU end 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf95ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf95ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf95ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf95ffb640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-14 : miss 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf95ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf967fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf967fc640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-53 info.flags=0x0 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf967fc640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-53 to cache LRU end 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf967fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13): adding source pipe: {s={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf967fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13): adding dest pipe: {s={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf967fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf967fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf967fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf95ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf95ffb640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-14 info.flags=0x0 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf95ffb640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-14 to cache LRU end 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf95ffb640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-14 : miss 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3959e140:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf95ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3959e140:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3959e140:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b3959e140:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b39529850:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39532dd0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b39529850:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 run: stack=0x558b39526660 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3952c110:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3952c110:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf95ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.396 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf95ffb640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-14 info.flags=0x0 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf95ffb640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-14 to cache LRU end 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf95ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19): adding source pipe: {s={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf95ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19): adding dest pipe: {s={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf95ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf95ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf95ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = fe55f868ea648d92b7aa0e065917a67b6f8253f099e166b22702a0b069553518 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:fe55f868ea648d92b7aa0e065917a67b6f8253f099e166b22702a0b069553518 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 71e8a468f631e32e7179d5bad5f6c62b9ea07d6438628794827e49e9ea6dd300 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=71e8a468f631e32e7179d5bad5f6c62b9ea07d6438628794827e49e9ea6dd300 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=6, curl_handle=0x558b39537db0 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 run: stack=0x558b39357a70 is io blocked 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b395996f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.397 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b395996f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b395996f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b395996f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdf88fe1640 20 link_request req_data=0x7fdf800062a0 req_data->id=6, curl_handle=0x558b39537db0 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 cr:s=0x558b39598bf0:op=0x558b3957de60:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 cr:s=0x558b39598bf0:op=0x558b3957d280:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 cr:s=0x558b39598bf0:op=0x558b3957d280:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = e1f36b4423450fb7fb556e60d74a25168cd2217b6c7b0280dc6bcde52e10d42a 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:e1f36b4423450fb7fb556e60d74a25168cd2217b6c7b0280dc6bcde52e10d42a 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 986a1b2c502f27fafca164ae371bf854013f5b0db51aa625636e63543b5762f4 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=986a1b2c502f27fafca164ae371bf854013f5b0db51aa625636e63543b5762f4 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=7, curl_handle=0x558b39534860 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.392+0000 7fdfe23ce900 20 run: stack=0x558b39598bf0 is io blocked 2026-03-20T11:47:08.398 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.393+0000 7fdf88fe1640 20 link_request req_data=0x7fdf7c005e00 req_data->id=7, curl_handle=0x558b39534860 2026-03-20T11:47:08.398 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.395+0000 7f769bc29640 -1 req 15211750561500224656 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596ee0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596ee0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b395b5470:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdfe23ce900 15 stack 0x558b3957acd0 end 2026-03-20T11:47:08.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdfe23ce900 20 run: stack=0x558b3957acd0 is done 2026-03-20T11:47:08.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39528580:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdfe23ce900 20 collect(): s=0x558b3943e140 stack=0x558b3957acd0 is complete 2026-03-20T11:47:08.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39528580:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdfe23ce900 15 stack 0x558b3943e140 end 2026-03-20T11:47:08.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is done 2026-03-20T11:47:08.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39528580:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:47:08.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959e140:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.421 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959e140:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdfe23ce900 20 enqueued request req=0x558b390d20e0 2026-03-20T11:47:08.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdfe23ce900 20 req: 0x558b390d20e0 2026-03-20T11:47:08.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is io blocked 2026-03-20T11:47:08.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdf957fa640 20 dequeued request req=0x558b390d20e0 2026-03-20T11:47:08.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdf957fa640 20 RGWWQ: empty 2026-03-20T11:47:08.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdf957fa640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 : miss 2026-03-20T11:47:08.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.423+0000 7fdf957fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.424+0000 7fdf957fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.424+0000 7fdf957fa640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 info.flags=0x17 2026-03-20T11:47:08.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.424+0000 7fdf957fa640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 to cache LRU end 2026-03-20T11:47:08.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.424+0000 7fdf957fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.424+0000 7fdf957fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.424+0000 7fdf957fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:47:08.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.424+0000 7fdf957fa640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-36 : miss 2026-03-20T11:47:08.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.424+0000 7fdf957fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.424+0000 7fdf957fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.424+0000 7fdf957fa640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-36 info.flags=0x0 2026-03-20T11:47:08.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.424+0000 7fdf957fa640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-36 to cache LRU end 2026-03-20T11:47:08.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.424+0000 7fdf957fa640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-36 : miss 2026-03-20T11:47:08.422 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.424+0000 7fdf957fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.424+0000 7fdf957fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.424+0000 7fdf957fa640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-36 info.flags=0x0 2026-03-20T11:47:08.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.424+0000 7fdf957fa640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-36 to cache LRU end 2026-03-20T11:47:08.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.424+0000 7fdf957fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41): adding source pipe: {s={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.424+0000 7fdf957fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41): adding dest pipe: {s={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.424+0000 7fdf957fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.424+0000 7fdf957fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.424+0000 7fdf957fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:47:08.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959e140:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959e140:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959e140:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959e140:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39528580:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596ee0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39528580:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.423 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 95fbdb81adf15068c7a3249982c205d0659331ad436340dc3eea9e9ae06dfa4e 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:95fbdb81adf15068c7a3249982c205d0659331ad436340dc3eea9e9ae06dfa4e 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = ab1264c386bdbf75703d5ddb1c5145a4ea0b0244974ad212acf304d86787b927 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=ab1264c386bdbf75703d5ddb1c5145a4ea0b0244974ad212acf304d86787b927 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=8, curl_handle=0x558b3953c0d0 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdfe23ce900 20 run: stack=0x558b3957acd0 is io blocked 2026-03-20T11:47:08.424 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.425+0000 7fdf88fe1640 20 link_request req_data=0x558b3953c7e0 req_data->id=8, curl_handle=0x558b3953c0d0 2026-03-20T11:47:08.424 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.427+0000 7f7682bf7640 -1 req 10682187775773608405 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.430 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957c750:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.430 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957c750:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.430 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3952b1e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.430 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdfe23ce900 15 stack 0x558b39540cc0 end 2026-03-20T11:47:08.430 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdfe23ce900 20 run: stack=0x558b39540cc0 is done 2026-03-20T11:47:08.430 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395272b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.430 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdfe23ce900 20 collect(): s=0x558b3952bf80 stack=0x558b39540cc0 is complete 2026-03-20T11:47:08.430 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.430 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.430 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.430 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395272b0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.430 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdfe23ce900 15 stack 0x558b3952bf80 end 2026-03-20T11:47:08.430 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.430 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is done 2026-03-20T11:47:08.430 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.430 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.430 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.430 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395272b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.430 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:47:08.430 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b39597510:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b39597510:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdfe23ce900 20 enqueued request req=0x558b395473f0 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdfe23ce900 20 req: 0x558b395473f0 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is io blocked 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdf94ff9640 20 dequeued request req=0x558b395473f0 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdf94ff9640 20 RGWWQ: empty 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdf94ff9640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 : miss 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdf94ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdf94ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdf94ff9640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 info.flags=0x17 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdf94ff9640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 to cache LRU end 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdf94ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdf94ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdf94ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdf94ff9640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-22 : miss 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdf94ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdf94ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdf94ff9640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-22 info.flags=0x0 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdf94ff9640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-22 to cache LRU end 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdf94ff9640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-22 : miss 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.431+0000 7fdf94ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdf94ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdf94ff9640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-22 info.flags=0x0 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdf94ff9640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-22 to cache LRU end 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdf94ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27): adding source pipe: {s={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdf94ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27): adding dest pipe: {s={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdf94ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdf94ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdf94ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b39597510:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b39597510:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b39597510:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b39597510:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395272b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3952b1e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.431 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395272b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957c720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957c720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = d076365970b30945af59d1d57e591becca6b4d7aa418da1ada443dafae709e4e 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:d076365970b30945af59d1d57e591becca6b4d7aa418da1ada443dafae709e4e 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 01fe486b005f5935432aceb047f6a4edfaa69314ad507abbed37b29bf5b6c624 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=01fe486b005f5935432aceb047f6a4edfaa69314ad507abbed37b29bf5b6c624 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=9, curl_handle=0x558b3952f650 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdfe23ce900 20 run: stack=0x558b39540cc0 is io blocked 2026-03-20T11:47:08.432 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.432+0000 7fdf88fe1640 20 link_request req_data=0x558b39530150 req_data->id=9, curl_handle=0x558b3952f650 2026-03-20T11:47:08.433 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.433+0000 7f7673bd9640 -1 req 1046194963198993771 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.433 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3952c110:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.433 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3952c110:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.433 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3952c110:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.433 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3952c110:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.433 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39532dd0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.433 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 15 stack 0x558b39357a70 end 2026-03-20T11:47:08.433 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 run: stack=0x558b39357a70 is done 2026-03-20T11:47:08.434 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b39529850:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.434 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 collect(): s=0x558b39526660 stack=0x558b39357a70 is complete 2026-03-20T11:47:08.434 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.434 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 10 trimming bilogs for bucket=:dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) markers=[00000000002.32.1,00000000002.59.1,00000000002.36.1,00000000002.31.1,00000000002.37.1,00000000002.34.1,00000000002.61.1,00000000002.37.1,00000000002.34.1,00000000002.53.1,00000000002.52.1], shards=11 2026-03-20T11:47:08.434 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b39532dd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:08.435 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 10 trimming bilog shard 0 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.32.1 2026-03-20T11:47:08.435 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 10 trimming bilog shard 1 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.59.1 2026-03-20T11:47:08.435 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 10 trimming bilog shard 2 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.36.1 2026-03-20T11:47:08.435 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 10 trimming bilog shard 3 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.31.1 2026-03-20T11:47:08.435 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 10 trimming bilog shard 4 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.37.1 2026-03-20T11:47:08.435 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 10 trimming bilog shard 5 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.34.1 2026-03-20T11:47:08.435 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 10 trimming bilog shard 6 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.61.1 2026-03-20T11:47:08.435 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 10 trimming bilog shard 7 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.37.1 2026-03-20T11:47:08.435 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 10 trimming bilog shard 8 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.34.1 2026-03-20T11:47:08.435 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 10 trimming bilog shard 9 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.53.1 2026-03-20T11:47:08.435 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 10 trimming bilog shard 10 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.52.1 2026-03-20T11:47:08.435 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 run: stack=0x558b39526660 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.435 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3957e8a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.435 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b390dbc30:op=0x558b390d1120:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.435 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b395338e0:op=0x558b390cb1b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.435 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b39533400:op=0x558b390ca490:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.435 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b395a1fc0:op=0x558b395246c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.435 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b39523a40:op=0x558b39523be0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.436 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b395323e0:op=0x558b3952d7f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.436 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b3953f300:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.436 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b3957c1c0:op=0x558b395419c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.436 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b3952ad30:op=0x558b395959e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.436 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39594e20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.436 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3957e8a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.436 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.0 2026-03-20T11:47:08.436 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 run: stack=0x558b39357a70 is io blocked 2026-03-20T11:47:08.436 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b390dbc30:op=0x558b390d1120:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.436 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.1 2026-03-20T11:47:08.436 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 run: stack=0x558b390dbc30 is io blocked 2026-03-20T11:47:08.436 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b395338e0:op=0x558b390cb1b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.436 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.2 2026-03-20T11:47:08.436 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 run: stack=0x558b395338e0 is io blocked 2026-03-20T11:47:08.436 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b39533400:op=0x558b390ca490:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.436 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.3 2026-03-20T11:47:08.436 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 run: stack=0x558b39533400 is io blocked 2026-03-20T11:47:08.436 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b395a1fc0:op=0x558b395246c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.436 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.4 2026-03-20T11:47:08.436 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 run: stack=0x558b395a1fc0 is io blocked 2026-03-20T11:47:08.440 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b39523a40:op=0x558b39523be0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.440 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.5 2026-03-20T11:47:08.440 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 run: stack=0x558b39523a40 is io blocked 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b395323e0:op=0x558b3952d7f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.6 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 run: stack=0x558b395323e0 is io blocked 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b3953f300:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.7 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 run: stack=0x558b39525130 is io blocked 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b3957c1c0:op=0x558b395419c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.8 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 run: stack=0x558b3957c1c0 is io blocked 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b3952ad30:op=0x558b395959e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.9 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 run: stack=0x558b3952ad30 is io blocked 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39594e20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.10 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.435+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is io blocked 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdfe23ce900 20 cr:s=0x558b39598bf0:op=0x558b3957d280:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdfe23ce900 20 cr:s=0x558b39598bf0:op=0x558b3957d280:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdfe23ce900 20 cr:s=0x558b39598bf0:op=0x558b3957de60:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdfe23ce900 15 stack 0x558b39598bf0 end 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdfe23ce900 20 run: stack=0x558b39598bf0 is done 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdfe23ce900 20 collect(): s=0x558b3943dd50 stack=0x558b39598bf0 is complete 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdfe23ce900 15 stack 0x558b3943dd50 end 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is done 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdfe23ce900 20 enqueued request req=0x558b39534d90 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdfe23ce900 20 req: 0x558b39534d90 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is io blocked 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdf947f8640 20 dequeued request req=0x558b39534d90 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdf947f8640 20 RGWWQ: empty 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdf947f8640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 : miss 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.436+0000 7fdf947f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b395a1fc0:op=0x558b395246c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b395a1fc0:op=0x558b395246c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 15 stack 0x558b395a1fc0 end 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 stack->operate() returned ret=-61 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 run: stack=0x558b395a1fc0 is done 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b39532dd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 run: stack=0x558b39526660 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b39523a40:op=0x558b39523be0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b39523a40:op=0x558b39523be0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 15 stack 0x558b39523a40 end 2026-03-20T11:47:08.441 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 stack->operate() returned ret=-61 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 run: stack=0x558b39523a40 is done 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b39532dd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 run: stack=0x558b39526660 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b395338e0:op=0x558b390cb1b0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b395338e0:op=0x558b390cb1b0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 15 stack 0x558b395338e0 end 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 stack->operate() returned ret=-61 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 run: stack=0x558b395338e0 is done 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b39532dd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 run: stack=0x558b39526660 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b3953f300:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b3953f300:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 15 stack 0x558b39525130 end 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 stack->operate() returned ret=-61 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 run: stack=0x558b39525130 is done 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b39532dd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 run: stack=0x558b39526660 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3957e8a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3957e8a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 15 stack 0x558b39357a70 end 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 stack->operate() returned ret=-61 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 run: stack=0x558b39357a70 is done 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b39532dd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 run: stack=0x558b39526660 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39594e20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39594e20:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 15 stack 0x558b395380a0 end 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 stack->operate() returned ret=-61 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is done 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b39532dd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 run: stack=0x558b39526660 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b3952ad30:op=0x558b395959e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b3952ad30:op=0x558b395959e0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 15 stack 0x558b3952ad30 end 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdf947f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 stack->operate() returned ret=-61 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 run: stack=0x558b3952ad30 is done 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b390dbc30:op=0x558b390d1120:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b390dbc30:op=0x558b390d1120:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 15 stack 0x558b390dbc30 end 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 stack->operate() returned ret=-61 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 run: stack=0x558b390dbc30 is done 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b39532dd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b39532dd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 run: stack=0x558b39526660 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdf947f8640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 info.flags=0x17 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdf947f8640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 to cache LRU end 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdf947f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdf947f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdf947f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdf947f8640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-25 : miss 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdf947f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b39533400:op=0x558b390ca490:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b39533400:op=0x558b390ca490:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 15 stack 0x558b39533400 end 2026-03-20T11:47:08.442 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 stack->operate() returned ret=-61 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 run: stack=0x558b39533400 is done 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b39532dd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 run: stack=0x558b39526660 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b395323e0:op=0x558b3952d7f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b395323e0:op=0x558b3952d7f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 15 stack 0x558b395323e0 end 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 stack->operate() returned ret=-61 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 run: stack=0x558b395323e0 is done 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b39532dd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.437+0000 7fdfe23ce900 20 run: stack=0x558b39526660 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.438+0000 7fdf947f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.438+0000 7fdf947f8640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-25 info.flags=0x0 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.438+0000 7fdf947f8640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-25 to cache LRU end 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.438+0000 7fdf947f8640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-25 : miss 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.438+0000 7fdf947f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 cr:s=0x558b3957c1c0:op=0x558b395419c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 cr:s=0x558b3957c1c0:op=0x558b395419c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 15 stack 0x558b3957c1c0 end 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 stack->operate() returned ret=-61 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 run: stack=0x558b3957c1c0 is done 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b39532dd0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 cr:s=0x558b39526660:op=0x558b39529850:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 trim: trimmed bucket instance dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 15 stack 0x558b39526660 end 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 run: stack=0x558b39526660 is done 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39597510:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39597510:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 enqueued request req=0x558b3952af20 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 req: 0x558b3952af20 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdf947f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdf947f8640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-25 info.flags=0x0 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdf947f8640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-25 to cache LRU end 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdf93ff7640 20 dequeued request req=0x558b3952af20 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdf93ff7640 20 RGWWQ: empty 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdf947f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30): adding source pipe: {s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdf947f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30): adding dest pipe: {s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdf93ff7640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 : miss 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdf93ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdf947f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdf947f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is io blocked 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdf947f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3957de60:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3957d280:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3957d280:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.443 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 832da1a4c5321d40045bc40c8075c5f306e05cdff261ebbe449203405690dd91 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:832da1a4c5321d40045bc40c8075c5f306e05cdff261ebbe449203405690dd91 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = d7560d2da006cd77ba3856d9a4bfdad2573aba2c8f7be96af57a3fe262d823cc 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=d7560d2da006cd77ba3856d9a4bfdad2573aba2c8f7be96af57a3fe262d823cc 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=10, curl_handle=0x558b39534860 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdfe23ce900 20 run: stack=0x558b39357a70 is io blocked 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdf88fe1640 20 link_request req_data=0x7fdf7c005e00 req_data->id=10, curl_handle=0x558b39534860 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdf93ff7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdf93ff7640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 info.flags=0x17 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdf93ff7640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 to cache LRU end 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdf93ff7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.444 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdf93ff7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.444 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.441+0000 7f76a9444640 -1 req 11587088745528980602 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdf93ff7640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:47:08.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdf93ff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:47:08.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdf93ff7640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-35 : miss 2026-03-20T11:47:08.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.439+0000 7fdf93ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdf93ff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdf93ff7640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-35 info.flags=0x0 2026-03-20T11:47:08.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdf93ff7640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-35 to cache LRU end 2026-03-20T11:47:08.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdf93ff7640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-35 : miss 2026-03-20T11:47:08.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdf93ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdf93ff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdf93ff7640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-35 info.flags=0x0 2026-03-20T11:47:08.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdf93ff7640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-35 to cache LRU end 2026-03-20T11:47:08.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdf93ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40): adding source pipe: {s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdf93ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40): adding dest pipe: {s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdf93ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.445 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdf93ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdf93ff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39597510:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39597510:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39597510:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39597510:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39597510:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39532dd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39532dd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 68ba62f49c4d3e0c8dd15ad9e0ed91f41b02e4e32598722c57f37e1840571237 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:68ba62f49c4d3e0c8dd15ad9e0ed91f41b02e4e32598722c57f37e1840571237 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.446 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.442+0000 7f76adc4d640 -1 req 2792362714747658235 0.002000003s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.446 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 7fd6034a123121207be25328fdfde40c1c9e7b80894d7ab1ae5291f93c86617e 2026-03-20T11:47:08.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=7fd6034a123121207be25328fdfde40c1c9e7b80894d7ab1ae5291f93c86617e 2026-03-20T11:47:08.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=11, curl_handle=0x558b39537db0 2026-03-20T11:47:08.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdfe23ce900 20 run: stack=0x558b39525130 is io blocked 2026-03-20T11:47:08.447 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.440+0000 7fdf88fe1640 20 link_request req_data=0x7fdf800062a0 req_data->id=11, curl_handle=0x558b39537db0 2026-03-20T11:47:08.468 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596ee0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdfe23ce900 15 stack 0x558b3957acd0 end 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdfe23ce900 20 run: stack=0x558b3957acd0 is done 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39528580:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdfe23ce900 20 collect(): s=0x558b3943e140 stack=0x558b3957acd0 is complete 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39528580:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdfe23ce900 15 stack 0x558b3943e140 end 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is done 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdfe23ce900 20 enqueued request req=0x558b390d20e0 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdfe23ce900 20 req: 0x558b390d20e0 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdf937f6640 20 dequeued request req=0x558b390d20e0 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdf937f6640 20 RGWWQ: empty 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdf937f6640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 : miss 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdf937f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is io blocked 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdf937f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdf937f6640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 info.flags=0x17 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdf937f6640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 to cache LRU end 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdf937f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdf937f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdf937f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdf937f6640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-26 : miss 2026-03-20T11:47:08.469 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.468+0000 7fdf937f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdf937f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdf937f6640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-26 info.flags=0x0 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdf937f6640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-26 to cache LRU end 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdf937f6640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-26 : miss 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdf937f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdf937f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdf937f6640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-26 info.flags=0x0 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdf937f6640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-26 to cache LRU end 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdf937f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31): adding source pipe: {s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdf937f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31): adding dest pipe: {s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdf937f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdf937f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdf937f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596e60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596e60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = c0b4f8feba5bbb3f4e23bae205087a2ae43d3b1f0fe8d139171d2300afc94525 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:c0b4f8feba5bbb3f4e23bae205087a2ae43d3b1f0fe8d139171d2300afc94525 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.470 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.471 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.471 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.471 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 34de69f83a575c32f28cc13ad7ffc1ca5ee53ef0b22a5678baba3757d98ac266 2026-03-20T11:47:08.471 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=34de69f83a575c32f28cc13ad7ffc1ca5ee53ef0b22a5678baba3757d98ac266 2026-03-20T11:47:08.471 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.471 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.471 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.471 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=12, curl_handle=0x558b3953c0d0 2026-03-20T11:47:08.471 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdfe23ce900 20 run: stack=0x558b3957acd0 is io blocked 2026-03-20T11:47:08.471 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.469+0000 7fdf88fe1640 20 link_request req_data=0x558b3953c7e0 req_data->id=12, curl_handle=0x558b3953c0d0 2026-03-20T11:47:08.471 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.471+0000 7f76b3458640 -1 req 18397792556800836693 0.002000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957c720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957c720:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3952b1e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdfe23ce900 15 stack 0x558b39540cc0 end 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdfe23ce900 20 run: stack=0x558b39540cc0 is done 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395272b0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdfe23ce900 20 collect(): s=0x558b3952bf80 stack=0x558b39540cc0 is complete 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395272b0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdfe23ce900 15 stack 0x558b3952bf80 end 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is done 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdfe23ce900 20 enqueued request req=0x558b395473f0 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdfe23ce900 20 req: 0x558b395473f0 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is io blocked 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdf92ff5640 20 dequeued request req=0x558b395473f0 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdf92ff5640 20 RGWWQ: empty 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdf92ff5640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 : miss 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdf92ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdf92ff5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdf92ff5640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 info.flags=0x17 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdf92ff5640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 to cache LRU end 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdf92ff5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdf92ff5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdf92ff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdf92ff5640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-51 : miss 2026-03-20T11:47:08.472 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.474+0000 7fdf92ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdf92ff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdf92ff5640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-51 info.flags=0x0 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdf92ff5640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-51 to cache LRU end 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdf92ff5640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-51 : miss 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdf92ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdf92ff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdf92ff5640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-51 info.flags=0x0 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdf92ff5640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-51 to cache LRU end 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdf92ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11): adding source pipe: {s={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdf92ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11): adding dest pipe: {s={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdf92ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdf92ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdf92ff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3952b1e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957c720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957c720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.473 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.474 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.474 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.474 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.474 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.474 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.474 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.474 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.474 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = cf05b36205810c936067313d174ec477d7fa95265ac806f9d1b22100e0cb1527 2026-03-20T11:47:08.474 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.474 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.474 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.474 INFO:teuthology.orchestra.run.vm07.stderr:cf05b36205810c936067313d174ec477d7fa95265ac806f9d1b22100e0cb1527 2026-03-20T11:47:08.474 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.474 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.474 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.474 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.474 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = d1ce2b8f228468ca00f3884c22e4f661fb005cee8fde4bb6b29494dcd6cbc585 2026-03-20T11:47:08.474 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=d1ce2b8f228468ca00f3884c22e4f661fb005cee8fde4bb6b29494dcd6cbc585 2026-03-20T11:47:08.474 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.474 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.474 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.474 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=13, curl_handle=0x558b3952f650 2026-03-20T11:47:08.474 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdfe23ce900 20 run: stack=0x558b39540cc0 is io blocked 2026-03-20T11:47:08.474 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.475+0000 7fdf88fe1640 20 link_request req_data=0x558b39530150 req_data->id=13, curl_handle=0x558b3952f650 2026-03-20T11:47:08.474 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.476+0000 7f76a0432640 -1 req 8369325432620015140 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3957d280:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3957d280:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3957de60:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdfe23ce900 15 stack 0x558b39357a70 end 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdfe23ce900 20 run: stack=0x558b39357a70 is done 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdfe23ce900 20 collect(): s=0x558b3943dd50 stack=0x558b39357a70 is complete 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdfe23ce900 15 stack 0x558b3943dd50 end 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is done 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdfe23ce900 20 enqueued request req=0x558b39534d90 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdfe23ce900 20 req: 0x558b39534d90 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdf927f4640 20 dequeued request req=0x558b39534d90 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdf927f4640 20 RGWWQ: empty 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdf927f4640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 : miss 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdf927f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is io blocked 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdf927f4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdf927f4640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 info.flags=0x17 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdf927f4640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 to cache LRU end 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdf927f4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdf927f4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdf927f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdf927f4640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-55 : miss 2026-03-20T11:47:08.480 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.482+0000 7fdf927f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdf927f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdf927f4640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-55 info.flags=0x0 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdf927f4640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-55 to cache LRU end 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdf927f4640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-55 : miss 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdf927f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39532dd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39532dd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39597510:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 15 stack 0x558b39525130 end 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 run: stack=0x558b39525130 is done 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 collect(): s=0x558b395380a0 stack=0x558b39525130 is complete 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 15 stack 0x558b395380a0 end 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is done 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39599680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39599680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 enqueued request req=0x558b3952af20 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 req: 0x558b3952af20 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is io blocked 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdf91ff3640 20 dequeued request req=0x558b3952af20 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdf91ff3640 20 RGWWQ: empty 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdf91ff3640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 : miss 2026-03-20T11:47:08.481 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdf91ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdf927f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdf927f4640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-55 info.flags=0x0 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdf927f4640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-55 to cache LRU end 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdf927f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding source pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdf927f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding dest pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdf91ff3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdf91ff3640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 info.flags=0x17 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdf91ff3640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 to cache LRU end 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdf91ff3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdf91ff3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdf927f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdf927f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdf91ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdf927f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdf91ff3640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-28 : miss 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdf91ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39532dd0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39536920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39536920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.482 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = c6ce7bdd642a7ff46d250c0c393b1e49a1c9b1b97d520e8fc63fc1eaf25561dd 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:c6ce7bdd642a7ff46d250c0c393b1e49a1c9b1b97d520e8fc63fc1eaf25561dd 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 808799074cec79aa3a253fd30dca669ec275eaafd59d7f1424c2afa2f1ade664 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=808799074cec79aa3a253fd30dca669ec275eaafd59d7f1424c2afa2f1ade664 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=14, curl_handle=0x558b39537db0 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdfe23ce900 20 run: stack=0x558b39525130 is io blocked 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.483+0000 7fdf88fe1640 20 link_request req_data=0x7fdf800062a0 req_data->id=14, curl_handle=0x558b39537db0 2026-03-20T11:47:08.483 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.484+0000 7f76593a4640 -1 req 2682242046189244486 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdf91ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdf91ff3640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-28 info.flags=0x0 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdf91ff3640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-28 to cache LRU end 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdf91ff3640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-28 : miss 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdf91ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdf91ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdf91ff3640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-28 info.flags=0x0 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdf91ff3640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-28 to cache LRU end 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdf91ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding source pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdf91ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding dest pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdf91ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdf91ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdf91ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:47:08.483 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39599680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39599680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39599680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39599680:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39599680:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3959b8a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3959b8a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 6fbcce6c71a91300500ddb2516cdd5deecab8f5dd07924a88c808640eeb9f11f 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:6fbcce6c71a91300500ddb2516cdd5deecab8f5dd07924a88c808640eeb9f11f 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 269f780392129b11042aaf3cc8233b830eb285033222682dcec1c0ff4c4d439a 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=269f780392129b11042aaf3cc8233b830eb285033222682dcec1c0ff4c4d439a 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=15, curl_handle=0x558b39534860 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdfe23ce900 20 run: stack=0x558b39357a70 is io blocked 2026-03-20T11:47:08.484 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.484+0000 7fdf88fe1640 20 link_request req_data=0x7fdf7c005e00 req_data->id=15, curl_handle=0x558b39534860 2026-03-20T11:47:08.484 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.485+0000 7f76c1c75640 -1 req 1535868546954664372 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596e60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596e60:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdfe23ce900 15 stack 0x558b3957acd0 end 2026-03-20T11:47:08.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdfe23ce900 20 run: stack=0x558b3957acd0 is done 2026-03-20T11:47:08.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdfe23ce900 20 collect(): s=0x558b3943e140 stack=0x558b3957acd0 is complete 2026-03-20T11:47:08.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdfe23ce900 15 stack 0x558b3943e140 end 2026-03-20T11:47:08.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is done 2026-03-20T11:47:08.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:47:08.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdfe23ce900 20 enqueued request req=0x558b390d20e0 2026-03-20T11:47:08.510 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdfe23ce900 20 req: 0x558b390d20e0 2026-03-20T11:47:08.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is io blocked 2026-03-20T11:47:08.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdf917f2640 20 dequeued request req=0x558b390d20e0 2026-03-20T11:47:08.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdf917f2640 20 RGWWQ: empty 2026-03-20T11:47:08.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdf917f2640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 : miss 2026-03-20T11:47:08.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdf917f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdf917f2640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdf917f2640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 info.flags=0x17 2026-03-20T11:47:08.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdf917f2640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 to cache LRU end 2026-03-20T11:47:08.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdf917f2640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdf917f2640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:47:08.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdf917f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:47:08.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdf917f2640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-31 : miss 2026-03-20T11:47:08.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.512+0000 7fdf917f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdf917f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdf917f2640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-31 info.flags=0x0 2026-03-20T11:47:08.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdf917f2640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-31 to cache LRU end 2026-03-20T11:47:08.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdf917f2640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-31 : miss 2026-03-20T11:47:08.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdf917f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdf917f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdf917f2640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-31 info.flags=0x0 2026-03-20T11:47:08.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdf917f2640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-31 to cache LRU end 2026-03-20T11:47:08.511 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdf917f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding source pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdf917f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding dest pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdf917f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdf917f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdf917f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596e40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596e40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 4cdfe4637b10153a5a76a68ac3d125811c4718e704ca6b267a83636f8ffe080b 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:4cdfe4637b10153a5a76a68ac3d125811c4718e704ca6b267a83636f8ffe080b 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = cf5d943bd4230571176dd08cb5626174c96545dda25278e5d4f04bdef38040fa 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=cf5d943bd4230571176dd08cb5626174c96545dda25278e5d4f04bdef38040fa 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=16, curl_handle=0x558b3953c0d0 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdfe23ce900 20 run: stack=0x558b3957acd0 is io blocked 2026-03-20T11:47:08.512 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.513+0000 7fdf88fe1640 20 link_request req_data=0x558b3953c7e0 req_data->id=16, curl_handle=0x558b3953c0d0 2026-03-20T11:47:08.512 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.514+0000 7f7699c25640 -1 req 2847706956103028504 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957c720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957c720:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3952b1e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdfe23ce900 15 stack 0x558b39540cc0 end 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdfe23ce900 20 run: stack=0x558b39540cc0 is done 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdfe23ce900 20 collect(): s=0x558b3952bf80 stack=0x558b39540cc0 is complete 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdfe23ce900 15 stack 0x558b3952bf80 end 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is done 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdfe23ce900 20 enqueued request req=0x558b395473f0 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdfe23ce900 20 req: 0x558b395473f0 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is io blocked 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdf90ff1640 20 dequeued request req=0x558b395473f0 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdf90ff1640 20 RGWWQ: empty 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdf90ff1640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 : miss 2026-03-20T11:47:08.515 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdf90ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdf90ff1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdf90ff1640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 info.flags=0x17 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdf90ff1640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 to cache LRU end 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdf90ff1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdf90ff1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdf90ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdf90ff1640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-39 : miss 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdf90ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdf90ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdf90ff1640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-39 info.flags=0x0 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdf90ff1640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-39 to cache LRU end 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdf90ff1640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-39 : miss 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdf90ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdf90ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdf90ff1640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-39 info.flags=0x0 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdf90ff1640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-39 to cache LRU end 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdf90ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding source pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdf90ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding dest pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdf90ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdf90ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.517+0000 7fdf90ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3952b1e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957c720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957c720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.516 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.517 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = a75c22e079c06ee32c8b83cabe26e26f7b86a2252f89e37fa67757f9863db1ea 2026-03-20T11:47:08.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.517 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.517 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.517 INFO:teuthology.orchestra.run.vm07.stderr:a75c22e079c06ee32c8b83cabe26e26f7b86a2252f89e37fa67757f9863db1ea 2026-03-20T11:47:08.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = c59f4c5fde8fa69ec210274f32f433e568b45990744c733c3943a859aba44f11 2026-03-20T11:47:08.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c59f4c5fde8fa69ec210274f32f433e568b45990744c733c3943a859aba44f11 2026-03-20T11:47:08.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=17, curl_handle=0x558b3952f650 2026-03-20T11:47:08.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdfe23ce900 20 run: stack=0x558b39540cc0 is io blocked 2026-03-20T11:47:08.517 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.518+0000 7fdf88fe1640 20 link_request req_data=0x558b39530150 req_data->id=17, curl_handle=0x558b3952f650 2026-03-20T11:47:08.517 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.518+0000 7f76f94e4640 -1 req 16661127112502204805 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39536920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39536920:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39532dd0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdfe23ce900 15 stack 0x558b39525130 end 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdfe23ce900 20 run: stack=0x558b39525130 is done 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdfe23ce900 20 collect(): s=0x558b3943dd50 stack=0x558b39525130 is complete 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdfe23ce900 15 stack 0x558b3943dd50 end 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is done 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39540e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39540e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdfe23ce900 20 enqueued request req=0x558b39534d90 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdfe23ce900 20 req: 0x558b39534d90 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is io blocked 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdf907f0640 20 dequeued request req=0x558b39534d90 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdf907f0640 20 RGWWQ: empty 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdf907f0640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 : miss 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdf907f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdf907f0640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdf907f0640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 info.flags=0x17 2026-03-20T11:47:08.523 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdf907f0640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 to cache LRU end 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdf907f0640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdf907f0640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdf907f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdf907f0640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-8 : miss 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.525+0000 7fdf907f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3959b8a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3959b8a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39599680:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 15 stack 0x558b39357a70 end 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 run: stack=0x558b39357a70 is done 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdf907f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdf907f0640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-8 info.flags=0x0 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdf907f0640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-8 to cache LRU end 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdf907f0640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-8 : miss 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdf907f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 collect(): s=0x558b395380a0 stack=0x558b39357a70 is complete 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 15 stack 0x558b395380a0 end 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is done 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 enqueued request req=0x558b3952af20 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 req: 0x558b3952af20 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is io blocked 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdf8ffef640 20 dequeued request req=0x558b3952af20 2026-03-20T11:47:08.524 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdf8ffef640 20 RGWWQ: empty 2026-03-20T11:47:08.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdf8ffef640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 : miss 2026-03-20T11:47:08.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdf8ffef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdf907f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdf907f0640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-8 info.flags=0x0 2026-03-20T11:47:08.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdf907f0640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-8 to cache LRU end 2026-03-20T11:47:08.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdf907f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding source pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdf907f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding dest pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdf907f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdf907f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdf907f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:47:08.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39540e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39540e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39540e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39540e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39532dd0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3959b8a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3959b8a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdf8ffef640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdf8ffef640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 info.flags=0x17 2026-03-20T11:47:08.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdf8ffef640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 to cache LRU end 2026-03-20T11:47:08.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdf8ffef640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.525 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdf8ffef640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdf8ffef640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.526+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.527+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 9c4bb316f21f7f1c2ebaed2452f29d755e8054f84530c6b6436a0d65af595484 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.527+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:9c4bb316f21f7f1c2ebaed2452f29d755e8054f84530c6b6436a0d65af595484 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.527+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.527+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.527+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.527+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.527+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 59edcac957c839e485b2468a5475f2da8ea0e405119aa26658d7b30637ebcc21 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.527+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=59edcac957c839e485b2468a5475f2da8ea0e405119aa26658d7b30637ebcc21 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.527+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.527+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.527+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.527+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=18, curl_handle=0x558b39534860 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.527+0000 7fdfe23ce900 20 run: stack=0x558b39357a70 is io blocked 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.527+0000 7fdf8ffef640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-16 : miss 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.527+0000 7fdf8ffef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.526 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.527+0000 7fdf88fe1640 20 link_request req_data=0x7fdf7c005e00 req_data->id=18, curl_handle=0x558b39534860 2026-03-20T11:47:08.526 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.527+0000 7f76c3478640 -1 req 12114803946886580041 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.527+0000 7fdf8ffef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.527+0000 7fdf8ffef640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-16 info.flags=0x0 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.527+0000 7fdf8ffef640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-16 to cache LRU end 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.527+0000 7fdf8ffef640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-16 : miss 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.527+0000 7fdf8ffef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdf8ffef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdf8ffef640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-16 info.flags=0x0 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdf8ffef640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-16 to cache LRU end 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdf8ffef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding source pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdf8ffef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding dest pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdf8ffef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdf8ffef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdf8ffef640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39536920:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39599680:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39536920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39536920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.527 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 0f0d07e2aa99ab32a9fb01f9a6a49c38294358ca1960ec9137e191dbaee80fad 2026-03-20T11:47:08.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.528 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.528 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.528 INFO:teuthology.orchestra.run.vm07.stderr:0f0d07e2aa99ab32a9fb01f9a6a49c38294358ca1960ec9137e191dbaee80fad 2026-03-20T11:47:08.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = bdf8f626e5d601465f8031a8015525455ba2467cb04c0cd8b6a00cb2613d6768 2026-03-20T11:47:08.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=bdf8f626e5d601465f8031a8015525455ba2467cb04c0cd8b6a00cb2613d6768 2026-03-20T11:47:08.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=19, curl_handle=0x558b39537db0 2026-03-20T11:47:08.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdfe23ce900 20 run: stack=0x558b39525130 is io blocked 2026-03-20T11:47:08.528 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.528+0000 7fdf88fe1640 20 link_request req_data=0x7fdf800062a0 req_data->id=19, curl_handle=0x558b39537db0 2026-03-20T11:47:08.528 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.529+0000 7f76dd4ac640 -1 req 5535133074828635664 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.553 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596e40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.556 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596e40:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdfe23ce900 15 stack 0x558b3957acd0 end 2026-03-20T11:47:08.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdfe23ce900 20 run: stack=0x558b3957acd0 is done 2026-03-20T11:47:08.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdfe23ce900 20 collect(): s=0x558b3943e140 stack=0x558b3957acd0 is complete 2026-03-20T11:47:08.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdfe23ce900 15 stack 0x558b3943e140 end 2026-03-20T11:47:08.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is done 2026-03-20T11:47:08.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:47:08.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdfe23ce900 20 enqueued request req=0x558b390d20e0 2026-03-20T11:47:08.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdfe23ce900 20 req: 0x558b390d20e0 2026-03-20T11:47:08.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is io blocked 2026-03-20T11:47:08.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdf8f7ee640 20 dequeued request req=0x558b390d20e0 2026-03-20T11:47:08.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdf8f7ee640 20 RGWWQ: empty 2026-03-20T11:47:08.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdf8f7ee640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 : miss 2026-03-20T11:47:08.557 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdf8f7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.558 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdf8f7ee640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:47:08.558 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdf8f7ee640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 info.flags=0x17 2026-03-20T11:47:08.558 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdf8f7ee640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 to cache LRU end 2026-03-20T11:47:08.558 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdf8f7ee640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.558 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdf8f7ee640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.558 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdf8f7ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:47:08.558 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdf8f7ee640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-43 : miss 2026-03-20T11:47:08.558 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.555+0000 7fdf8f7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.558 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdf8f7ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.558 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdf8f7ee640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-43 info.flags=0x0 2026-03-20T11:47:08.558 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdf8f7ee640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-43 to cache LRU end 2026-03-20T11:47:08.558 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdf8f7ee640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-43 : miss 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdf8f7ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdf8f7ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdf8f7ee640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-43 info.flags=0x0 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdf8f7ee640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-43 to cache LRU end 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdf8f7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding source pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdf8f7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding dest pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdf8f7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdf8f7ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdf8f7ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596e40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596e40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 04a088adbb342d6bef4ff75755e86c4e35f250f7fc66fec3b2d0cbc739c8db95 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:04a088adbb342d6bef4ff75755e86c4e35f250f7fc66fec3b2d0cbc739c8db95 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 0ce191d6eadf848c7d5128ed98502f87fac1fb0685a1ce24d3a476b261bed4a4 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=0ce191d6eadf848c7d5128ed98502f87fac1fb0685a1ce24d3a476b261bed4a4 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=20, curl_handle=0x558b3953c0d0 2026-03-20T11:47:08.559 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.556+0000 7fdf88fe1640 20 link_request req_data=0x558b3953c7e0 req_data->id=20, curl_handle=0x558b3953c0d0 2026-03-20T11:47:08.559 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.557+0000 7f76abc49640 -1 req 9766670946768257312 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.560 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.557+0000 7fdfe23ce900 20 run: stack=0x558b3957acd0 is io blocked 2026-03-20T11:47:08.560 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957c720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.560 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957c720:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.560 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3952b1e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.560 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdfe23ce900 15 stack 0x558b39540cc0 end 2026-03-20T11:47:08.560 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdfe23ce900 20 run: stack=0x558b39540cc0 is done 2026-03-20T11:47:08.560 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.560 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdfe23ce900 20 collect(): s=0x558b3952bf80 stack=0x558b39540cc0 is complete 2026-03-20T11:47:08.560 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.560 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.560 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.560 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.560 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdfe23ce900 15 stack 0x558b3952bf80 end 2026-03-20T11:47:08.560 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.560 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is done 2026-03-20T11:47:08.560 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.560 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.560 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.560 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.560 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:47:08.560 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.560 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.560 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdfe23ce900 20 enqueued request req=0x558b395473f0 2026-03-20T11:47:08.560 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.560 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdfe23ce900 20 req: 0x558b395473f0 2026-03-20T11:47:08.560 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is io blocked 2026-03-20T11:47:08.560 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdf8efed640 20 dequeued request req=0x558b395473f0 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdf8efed640 20 RGWWQ: empty 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdf8efed640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 : miss 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdf8efed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdf8efed640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdf8efed640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 info.flags=0x17 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdf8efed640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 to cache LRU end 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdf8efed640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdf8efed640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdf8efed640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdf8efed640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-10 : miss 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.559+0000 7fdf8efed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdf8efed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdf8efed640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-10 info.flags=0x0 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdf8efed640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-10 to cache LRU end 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdf8efed640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-10 : miss 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdf8efed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdf8efed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdf8efed640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-10 info.flags=0x0 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdf8efed640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-10 to cache LRU end 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdf8efed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding source pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdf8efed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding dest pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdf8efed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdf8efed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdf8efed640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3952b1e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957c720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957c720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 1b3753969f72f695a5c8597ff634cea5e7221ad112b275d5b53776acc2a75550 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:1b3753969f72f695a5c8597ff634cea5e7221ad112b275d5b53776acc2a75550 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.561 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = d4d1ebfb10cd3524eec1adcc81f27d0f32bf55ecdd01ee69affc39d8b5622783 2026-03-20T11:47:08.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=d4d1ebfb10cd3524eec1adcc81f27d0f32bf55ecdd01ee69affc39d8b5622783 2026-03-20T11:47:08.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=21, curl_handle=0x558b3952f650 2026-03-20T11:47:08.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdf88fe1640 20 link_request req_data=0x558b39530150 req_data->id=21, curl_handle=0x558b3952f650 2026-03-20T11:47:08.562 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.561+0000 7f76a7c41640 -1 req 2570816204165429615 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.562 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.560+0000 7fdfe23ce900 20 run: stack=0x558b39540cc0 is io blocked 2026-03-20T11:47:08.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3959b8a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3959b8a0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39532dd0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdfe23ce900 15 stack 0x558b39357a70 end 2026-03-20T11:47:08.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdfe23ce900 20 run: stack=0x558b39357a70 is done 2026-03-20T11:47:08.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdfe23ce900 20 collect(): s=0x558b3943dd50 stack=0x558b39357a70 is complete 2026-03-20T11:47:08.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdfe23ce900 15 stack 0x558b3943dd50 end 2026-03-20T11:47:08.566 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is done 2026-03-20T11:47:08.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:47:08.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39540e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39540e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdfe23ce900 20 enqueued request req=0x558b39534d90 2026-03-20T11:47:08.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdfe23ce900 20 req: 0x558b39534d90 2026-03-20T11:47:08.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is io blocked 2026-03-20T11:47:08.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdf8e7ec640 20 dequeued request req=0x558b39534d90 2026-03-20T11:47:08.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdf8e7ec640 20 RGWWQ: empty 2026-03-20T11:47:08.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdf8e7ec640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 : miss 2026-03-20T11:47:08.567 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdf8e7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.568 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdf8e7ec640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:47:08.568 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdf8e7ec640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 info.flags=0x17 2026-03-20T11:47:08.568 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdf8e7ec640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 to cache LRU end 2026-03-20T11:47:08.568 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdf8e7ec640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.568 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdf8e7ec640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.568 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdf8e7ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:47:08.568 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdf8e7ec640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-3 : miss 2026-03-20T11:47:08.568 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdf8e7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.568 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdf8e7ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.568 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdf8e7ec640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-3 info.flags=0x0 2026-03-20T11:47:08.568 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdf8e7ec640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-3 to cache LRU end 2026-03-20T11:47:08.568 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdf8e7ec640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-3 : miss 2026-03-20T11:47:08.568 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.568+0000 7fdf8e7ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.568 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdf8e7ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.568 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdf8e7ec640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-3 info.flags=0x0 2026-03-20T11:47:08.568 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdf8e7ec640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-3 to cache LRU end 2026-03-20T11:47:08.568 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdf8e7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding source pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdf8e7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding dest pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdf8e7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdf8e7ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdf8e7ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39540e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39540e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39540e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39540e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3959b8a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39532dd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39532dd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 45d96989fa11bca2f3fe8b3d1be98401747fbecea48cb6b1e05aa09b3d2a296b 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.569 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.570 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.570 INFO:teuthology.orchestra.run.vm07.stderr:45d96989fa11bca2f3fe8b3d1be98401747fbecea48cb6b1e05aa09b3d2a296b 2026-03-20T11:47:08.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 557fbc5e758335b551841bf2d6bdc347a342df18033c32ad86b02a53e4b16449 2026-03-20T11:47:08.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=557fbc5e758335b551841bf2d6bdc347a342df18033c32ad86b02a53e4b16449 2026-03-20T11:47:08.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=22, curl_handle=0x558b39534860 2026-03-20T11:47:08.570 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.569+0000 7fdf88fe1640 20 link_request req_data=0x7fdf7c005e00 req_data->id=22, curl_handle=0x558b39534860 2026-03-20T11:47:08.570 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.569+0000 7f76dccab640 -1 req 5075765812042734139 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdfe23ce900 20 run: stack=0x558b39357a70 is io blocked 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39536920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39536920:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39599680:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdfe23ce900 15 stack 0x558b39525130 end 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdfe23ce900 20 run: stack=0x558b39525130 is done 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdfe23ce900 20 collect(): s=0x558b395380a0 stack=0x558b39525130 is complete 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdfe23ce900 15 stack 0x558b395380a0 end 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is done 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b3953f300:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b3953f300:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdfe23ce900 20 enqueued request req=0x558b3952af20 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdfe23ce900 20 req: 0x558b3952af20 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdf8dfeb640 20 dequeued request req=0x558b3952af20 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdf8dfeb640 20 RGWWQ: empty 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdf8dfeb640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 : miss 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdf8dfeb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is io blocked 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdf8dfeb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdf8dfeb640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 info.flags=0x17 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdf8dfeb640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 to cache LRU end 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdf8dfeb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdf8dfeb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdf8dfeb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdf8dfeb640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-9 : miss 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdf8dfeb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdf8dfeb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdf8dfeb640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-9 info.flags=0x0 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdf8dfeb640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-9 to cache LRU end 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdf8dfeb640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-9 : miss 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.570+0000 7fdf8dfeb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdf8dfeb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdf8dfeb640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-9 info.flags=0x0 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdf8dfeb640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-9 to cache LRU end 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdf8dfeb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding source pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdf8dfeb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding dest pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.571 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdf8dfeb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdf8dfeb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdf8dfeb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b3953f300:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b3953f300:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b3953f300:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b3953f300:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39536920:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b3953f300:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b3953f300:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = e64183d8fd7d86c2a39b4e390812beec172eeaeb39e276dfd9c67fbe095cedc5 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:e64183d8fd7d86c2a39b4e390812beec172eeaeb39e276dfd9c67fbe095cedc5 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 588664bec76f20fc42acd6c7b5188fd9f36a1eba16cffe58b832abd5550087aa 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=588664bec76f20fc42acd6c7b5188fd9f36a1eba16cffe58b832abd5550087aa 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=23, curl_handle=0x558b39537db0 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdf88fe1640 20 link_request req_data=0x7fdf800062a0 req_data->id=23, curl_handle=0x558b39537db0 2026-03-20T11:47:08.572 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.571+0000 7fdfe23ce900 20 run: stack=0x558b39525130 is io blocked 2026-03-20T11:47:08.572 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.571+0000 7f7697c21640 -1 req 12864718943496764289 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.596 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596e40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.596 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596e40:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.596 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.596 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdfe23ce900 15 stack 0x558b3957acd0 end 2026-03-20T11:47:08.596 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdfe23ce900 20 run: stack=0x558b3957acd0 is done 2026-03-20T11:47:08.596 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.596 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdfe23ce900 20 collect(): s=0x558b3943e140 stack=0x558b3957acd0 is complete 2026-03-20T11:47:08.596 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.596 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.597 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.597 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.597 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdfe23ce900 15 stack 0x558b3943e140 end 2026-03-20T11:47:08.597 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.597 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is done 2026-03-20T11:47:08.597 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.597 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.597 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.597 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.597 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:47:08.597 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.597 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.597 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdfe23ce900 20 enqueued request req=0x558b390d20e0 2026-03-20T11:47:08.597 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.597 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdfe23ce900 20 req: 0x558b390d20e0 2026-03-20T11:47:08.597 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdf8d7ea640 20 dequeued request req=0x558b390d20e0 2026-03-20T11:47:08.597 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdf8d7ea640 20 RGWWQ: empty 2026-03-20T11:47:08.597 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdf8d7ea640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 : miss 2026-03-20T11:47:08.597 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdf8d7ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.597 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is io blocked 2026-03-20T11:47:08.598 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdf8d7ea640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.598 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdf8d7ea640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 info.flags=0x17 2026-03-20T11:47:08.598 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.598+0000 7fdf8d7ea640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 to cache LRU end 2026-03-20T11:47:08.598 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdf8d7ea640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.598 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdf8d7ea640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.598 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdf8d7ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:47:08.598 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdf8d7ea640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-38 : miss 2026-03-20T11:47:08.598 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdf8d7ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.598 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdf8d7ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.598 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdf8d7ea640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-38 info.flags=0x0 2026-03-20T11:47:08.598 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdf8d7ea640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-38 to cache LRU end 2026-03-20T11:47:08.598 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdf8d7ea640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-38 : miss 2026-03-20T11:47:08.598 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdf8d7ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.598 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdf8d7ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.598 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdf8d7ea640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-38 info.flags=0x0 2026-03-20T11:47:08.598 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdf8d7ea640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-38 to cache LRU end 2026-03-20T11:47:08.598 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdf8d7ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding source pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.598 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdf8d7ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding dest pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.598 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdf8d7ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.598 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdf8d7ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.598 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdf8d7ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:47:08.598 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.598 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.598 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596e40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596e40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.599+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.600+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 7df20d72f30aef219a6958d9f0141e22e07f22eda0fdbc04bee2748decae0651 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.600+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:7df20d72f30aef219a6958d9f0141e22e07f22eda0fdbc04bee2748decae0651 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.600+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.600+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.600+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.600+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.600+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 1ac8f501a011fb9dee774d79299c1fd273b512f2fdad707860ca482236c18388 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.600+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=1ac8f501a011fb9dee774d79299c1fd273b512f2fdad707860ca482236c18388 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.600+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.600+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.600+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.600+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=24, curl_handle=0x558b3953c0d0 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.600+0000 7fdfe23ce900 20 run: stack=0x558b3957acd0 is io blocked 2026-03-20T11:47:08.599 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.600+0000 7fdf88fe1640 20 link_request req_data=0x558b3953c7e0 req_data->id=24, curl_handle=0x558b3953c0d0 2026-03-20T11:47:08.599 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.600+0000 7f76d3c99640 -1 req 624047362216601722 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957c720:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957c720:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3952b1e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdfe23ce900 15 stack 0x558b39540cc0 end 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdfe23ce900 20 run: stack=0x558b39540cc0 is done 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdfe23ce900 20 collect(): s=0x558b3952bf80 stack=0x558b39540cc0 is complete 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdfe23ce900 15 stack 0x558b3952bf80 end 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is done 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdfe23ce900 20 enqueued request req=0x558b395473f0 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdfe23ce900 20 req: 0x558b395473f0 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdf8cfe9640 20 dequeued request req=0x558b395473f0 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdf8cfe9640 20 RGWWQ: empty 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdf8cfe9640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 : miss 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdf8cfe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.600 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is io blocked 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdf8cfe9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdf8cfe9640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 info.flags=0x17 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdf8cfe9640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 to cache LRU end 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdf8cfe9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdf8cfe9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdf8cfe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdf8cfe9640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-15 : miss 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdf8cfe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdf8cfe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdf8cfe9640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-15 info.flags=0x0 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdf8cfe9640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-15 to cache LRU end 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdf8cfe9640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-15 : miss 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.602+0000 7fdf8cfe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdf8cfe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdf8cfe9640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-15 info.flags=0x0 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdf8cfe9640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-15 to cache LRU end 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdf8cfe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding source pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdf8cfe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding dest pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdf8cfe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdf8cfe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdf8cfe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957c720:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3952b1e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3952b1e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.601 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = d5c7a325591fc7951baf160bb4bcda921c219064d69c02df343ca84bd2ea2793 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:d5c7a325591fc7951baf160bb4bcda921c219064d69c02df343ca84bd2ea2793 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 20304147fecade135d242ad47cce4a28e84699055edfe1e8f8bf677bdcf64c42 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=20304147fecade135d242ad47cce4a28e84699055edfe1e8f8bf677bdcf64c42 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=25, curl_handle=0x558b3952f650 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdfe23ce900 20 run: stack=0x558b39540cc0 is io blocked 2026-03-20T11:47:08.602 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.603+0000 7fdf88fe1640 20 link_request req_data=0x558b39530150 req_data->id=25, curl_handle=0x558b3952f650 2026-03-20T11:47:08.602 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.604+0000 7f76e0cb3640 -1 req 16998716556722476750 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39532dd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39532dd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3959b8a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdfe23ce900 15 stack 0x558b39357a70 end 2026-03-20T11:47:08.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdfe23ce900 20 run: stack=0x558b39357a70 is done 2026-03-20T11:47:08.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdfe23ce900 20 collect(): s=0x558b3943dd50 stack=0x558b39357a70 is complete 2026-03-20T11:47:08.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.608 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdfe23ce900 15 stack 0x558b3943dd50 end 2026-03-20T11:47:08.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is done 2026-03-20T11:47:08.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:47:08.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39540e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39540e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdfe23ce900 20 enqueued request req=0x558b39534d90 2026-03-20T11:47:08.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdfe23ce900 20 req: 0x558b39534d90 2026-03-20T11:47:08.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is io blocked 2026-03-20T11:47:08.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdf9effd640 20 dequeued request req=0x558b39534d90 2026-03-20T11:47:08.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdf9effd640 20 RGWWQ: empty 2026-03-20T11:47:08.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdf9effd640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 : miss 2026-03-20T11:47:08.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdf9effd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdf9effd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.609 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdf9effd640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 info.flags=0x17 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdf9effd640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 to cache LRU end 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdf9effd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdf9effd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdf9effd640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdf9effd640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-13 : miss 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.610+0000 7fdf9effd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdf9effd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdf9effd640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-13 info.flags=0x0 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdf9effd640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-13 to cache LRU end 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdf9effd640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-13 : miss 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdf9effd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdf9effd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdf9effd640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-13 info.flags=0x0 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdf9effd640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-13 to cache LRU end 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdf9effd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18): adding source pipe: {s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdf9effd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18): adding dest pipe: {s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdf9effd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdf9effd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdf9effd640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39540e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39540e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39540e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39540e00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39532dd0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39540e00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39540e00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = ea16d1f5c3f668da350abe1d23fd573078124f2abd6d3cdc607dc8b235f1bc28 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.610 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.611 INFO:teuthology.orchestra.run.vm07.stderr:ea16d1f5c3f668da350abe1d23fd573078124f2abd6d3cdc607dc8b235f1bc28 2026-03-20T11:47:08.611 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.611 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.611 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.611 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.611 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = f170a762dfac692f1fc79fa3e40c7f31b23a3f4bd20cf03974e862b723eb2612 2026-03-20T11:47:08.611 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=f170a762dfac692f1fc79fa3e40c7f31b23a3f4bd20cf03974e862b723eb2612 2026-03-20T11:47:08.611 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.611 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.611 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-13%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.611 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=26, curl_handle=0x558b39534860 2026-03-20T11:47:08.611 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdfe23ce900 20 run: stack=0x558b39357a70 is io blocked 2026-03-20T11:47:08.611 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.611+0000 7fdf88fe1640 20 link_request req_data=0x7fdf7c005e00 req_data->id=26, curl_handle=0x558b39534860 2026-03-20T11:47:08.611 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.612+0000 7f76e6cbf640 -1 req 9959668747389090083 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b3953f300:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b3953f300:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39536920:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfe23ce900 15 stack 0x558b39525130 end 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfe23ce900 20 run: stack=0x558b39525130 is done 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfe23ce900 20 collect(): s=0x558b395380a0 stack=0x558b39525130 is complete 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfe23ce900 15 stack 0x558b395380a0 end 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is done 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39541ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39541ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfe23ce900 20 enqueued request req=0x558b3952af20 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfe23ce900 20 req: 0x558b3952af20 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is io blocked 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfa4ff9640 20 dequeued request req=0x558b3952af20 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfa4ff9640 20 RGWWQ: empty 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfa4ff9640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 : miss 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.612+0000 7fdfa4ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfa4ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfa4ff9640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 info.flags=0x17 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfa4ff9640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 to cache LRU end 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfa4ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfa4ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfa4ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfa4ff9640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-11 : miss 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfa4ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfa4ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.612 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfa4ff9640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-11 info.flags=0x0 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfa4ff9640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-11 to cache LRU end 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfa4ff9640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-11 : miss 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfa4ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfa4ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfa4ff9640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-11 info.flags=0x0 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfa4ff9640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-11 to cache LRU end 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfa4ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16): adding source pipe: {s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfa4ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16): adding dest pipe: {s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.613 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.614+0000 7f76793e4640 -1 req 2504205636410318597 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfa4ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfa4ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfa4ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39541ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39541ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39541ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39541ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39536920:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b3953f300:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b3953f300:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = c5ec18b9aff4939ef8298d749f2921432afdefbe5edcf17873df87d4ffe6ea83 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:c5ec18b9aff4939ef8298d749f2921432afdefbe5edcf17873df87d4ffe6ea83 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.613 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.614 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.614 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.614 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 075030ce766dd59cc82f374e097326cffe06cf385f56d43584f7eecf34ee4482 2026-03-20T11:47:08.614 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=075030ce766dd59cc82f374e097326cffe06cf385f56d43584f7eecf34ee4482 2026-03-20T11:47:08.614 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.614 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.614 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-11%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.614 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=27, curl_handle=0x558b39537db0 2026-03-20T11:47:08.614 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdfe23ce900 20 run: stack=0x558b39525130 is io blocked 2026-03-20T11:47:08.614 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.613+0000 7fdf88fe1640 20 link_request req_data=0x7fdf800062a0 req_data->id=27, curl_handle=0x558b39537db0 2026-03-20T11:47:08.639 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596e40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.639 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596e40:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.639 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.639 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdfe23ce900 15 stack 0x558b3957acd0 end 2026-03-20T11:47:08.639 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdfe23ce900 20 run: stack=0x558b3957acd0 is done 2026-03-20T11:47:08.639 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.639 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdfe23ce900 20 collect(): s=0x558b3943e140 stack=0x558b3957acd0 is complete 2026-03-20T11:47:08.639 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.639 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.639 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.639 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.639 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdfe23ce900 15 stack 0x558b3943e140 end 2026-03-20T11:47:08.639 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.639 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is done 2026-03-20T11:47:08.639 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.639 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.639 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.639 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.639 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:47:08.639 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.639 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdfe23ce900 20 enqueued request req=0x558b390d20e0 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdfe23ce900 20 req: 0x558b390d20e0 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is io blocked 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdf8c7e8640 20 dequeued request req=0x558b390d20e0 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdf8c7e8640 20 RGWWQ: empty 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdf8c7e8640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 : miss 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdf8c7e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdf8c7e8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdf8c7e8640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 info.flags=0x17 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdf8c7e8640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 to cache LRU end 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdf8c7e8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdf8c7e8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdf8c7e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdf8c7e8640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-12 : miss 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.641+0000 7fdf8c7e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdf8c7e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdf8c7e8640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-12 info.flags=0x0 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdf8c7e8640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-12 to cache LRU end 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdf8c7e8640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-12 : miss 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdf8c7e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdf8c7e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdf8c7e8640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-12 info.flags=0x0 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdf8c7e8640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-12 to cache LRU end 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdf8c7e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17): adding source pipe: {s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdf8c7e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17): adding dest pipe: {s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdf8c7e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdf8c7e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdf8c7e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.640 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596e40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596e40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = f76fcaf6dc76103f1b537099471ae41bf6a3e715ccff18a652fea9f2b7469265 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:f76fcaf6dc76103f1b537099471ae41bf6a3e715ccff18a652fea9f2b7469265 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 7208caf76a6ed58b87c81b33c79e505196e73d44b7a5c17a0d0e418606e8b10d 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=7208caf76a6ed58b87c81b33c79e505196e73d44b7a5c17a0d0e418606e8b10d 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-12%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=28, curl_handle=0x558b3953c0d0 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdfe23ce900 20 run: stack=0x558b3957acd0 is io blocked 2026-03-20T11:47:08.641 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.642+0000 7fdf88fe1640 20 link_request req_data=0x558b3953c7e0 req_data->id=28, curl_handle=0x558b3953c0d0 2026-03-20T11:47:08.641 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.643+0000 7f76dfcb1640 -1 req 6975965778097830703 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.643 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3952b1e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.644 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3952b1e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.644 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957c720:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.644 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdfe23ce900 15 stack 0x558b39540cc0 end 2026-03-20T11:47:08.644 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdfe23ce900 20 run: stack=0x558b39540cc0 is done 2026-03-20T11:47:08.644 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.644 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdfe23ce900 20 collect(): s=0x558b3952bf80 stack=0x558b39540cc0 is complete 2026-03-20T11:47:08.644 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.644 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.644 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.644 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.644 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdfe23ce900 15 stack 0x558b3952bf80 end 2026-03-20T11:47:08.644 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.644 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is done 2026-03-20T11:47:08.644 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.644 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.644 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.644 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.644 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:47:08.644 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.644 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.644 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdfe23ce900 20 enqueued request req=0x558b395473f0 2026-03-20T11:47:08.644 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.644 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdfe23ce900 20 req: 0x558b395473f0 2026-03-20T11:47:08.644 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is io blocked 2026-03-20T11:47:08.644 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdf9ffff640 20 dequeued request req=0x558b395473f0 2026-03-20T11:47:08.644 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdf9ffff640 20 RGWWQ: empty 2026-03-20T11:47:08.644 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdf9ffff640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 : miss 2026-03-20T11:47:08.644 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdf9ffff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdf9ffff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdf9ffff640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 info.flags=0x17 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdf9ffff640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 to cache LRU end 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdf9ffff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdf9ffff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdf9ffff640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdf9ffff640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-40 : miss 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdf9ffff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdf9ffff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdf9ffff640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-40 info.flags=0x0 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdf9ffff640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-40 to cache LRU end 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdf9ffff640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-40 : miss 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.645+0000 7fdf9ffff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdf9ffff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdf9ffff640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-40 info.flags=0x0 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdf9ffff640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-40 to cache LRU end 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdf9ffff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1): adding source pipe: {s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdf9ffff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1): adding dest pipe: {s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdf9ffff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdf9ffff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdf9ffff640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3959b8a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3952b1e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3952b1e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.645 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.646 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.646 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.646 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.646 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.646 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.646 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.646 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.646 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.646 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = dc1e1cd05b5ff12f9cca29f6299e5c28a3d7cf7eebd96c3f1d88301bdb2eea54 2026-03-20T11:47:08.646 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.646 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.646 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.646 INFO:teuthology.orchestra.run.vm07.stderr:dc1e1cd05b5ff12f9cca29f6299e5c28a3d7cf7eebd96c3f1d88301bdb2eea54 2026-03-20T11:47:08.646 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.646 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.646 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.646 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.646 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = ec2cc406656332c552eb9186d67e9794440c9441e7aa152546683ad344bd6886 2026-03-20T11:47:08.646 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=ec2cc406656332c552eb9186d67e9794440c9441e7aa152546683ad344bd6886 2026-03-20T11:47:08.646 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.646 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.646 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-40%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.646 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=29, curl_handle=0x558b3952f650 2026-03-20T11:47:08.646 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdfe23ce900 20 run: stack=0x558b39540cc0 is io blocked 2026-03-20T11:47:08.646 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.646+0000 7fdf88fe1640 20 link_request req_data=0x558b39530150 req_data->id=29, curl_handle=0x558b3952f650 2026-03-20T11:47:08.646 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.646+0000 7f7705cfd640 -1 req 14603064207747359933 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39540e00:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39540e00:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39532dd0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdfe23ce900 15 stack 0x558b39357a70 end 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdfe23ce900 20 run: stack=0x558b39357a70 is done 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdfe23ce900 20 collect(): s=0x558b3943dd50 stack=0x558b39357a70 is complete 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdfe23ce900 15 stack 0x558b3943dd50 end 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is done 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b390cf970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b390cf970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdfe23ce900 20 enqueued request req=0x558b39534d90 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdfe23ce900 20 req: 0x558b39534d90 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is io blocked 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdf9f7fe640 20 dequeued request req=0x558b39534d90 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdf9f7fe640 20 RGWWQ: empty 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdf9f7fe640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 : miss 2026-03-20T11:47:08.651 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdf9f7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdf9f7fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdf9f7fe640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 info.flags=0x17 2026-03-20T11:47:08.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdf9f7fe640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 to cache LRU end 2026-03-20T11:47:08.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdf9f7fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdf9f7fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdf9f7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:47:08.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdf9f7fe640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-18 : miss 2026-03-20T11:47:08.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdf9f7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.653+0000 7fdf9f7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdf9f7fe640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-18 info.flags=0x0 2026-03-20T11:47:08.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdf9f7fe640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-18 to cache LRU end 2026-03-20T11:47:08.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdf9f7fe640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-18 : miss 2026-03-20T11:47:08.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdf9f7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdf9f7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdf9f7fe640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-18 info.flags=0x0 2026-03-20T11:47:08.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdf9f7fe640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-18 to cache LRU end 2026-03-20T11:47:08.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdf9f7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23): adding source pipe: {s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdf9f7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23): adding dest pipe: {s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdf9f7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdf9f7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdf9f7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23 2026-03-20T11:47:08.652 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b390cf970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b390cf970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b390cf970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b390cf970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39540e00:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39532dd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39532dd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 4bd2972d4b2c6779dd7d4e2fc4e1d45ee4456cb9bbcaa9fa375d2c7a602248e2 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:4bd2972d4b2c6779dd7d4e2fc4e1d45ee4456cb9bbcaa9fa375d2c7a602248e2 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = d2dd74dccc20b2ac9a278f99ff284409592236bbc023cc72b5e74320877affec 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=d2dd74dccc20b2ac9a278f99ff284409592236bbc023cc72b5e74320877affec 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-18%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.653 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=30, curl_handle=0x558b39534860 2026-03-20T11:47:08.654 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.655+0000 7f76ddcad640 -1 req 2122560044299008740 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.654 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdfe23ce900 20 run: stack=0x558b39357a70 is io blocked 2026-03-20T11:47:08.654 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.654+0000 7fdf88fe1640 20 link_request req_data=0x7fdf7c005e00 req_data->id=30, curl_handle=0x558b39534860 2026-03-20T11:47:08.654 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b3953f300:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.654 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b3953f300:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.654 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39536920:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.654 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdfe23ce900 15 stack 0x558b39525130 end 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdfe23ce900 20 run: stack=0x558b39525130 is done 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdfe23ce900 20 collect(): s=0x558b395380a0 stack=0x558b39525130 is complete 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdfe23ce900 15 stack 0x558b395380a0 end 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is done 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39541ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39541ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdfe23ce900 20 enqueued request req=0x558b3952af20 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdfe23ce900 20 req: 0x558b3952af20 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is io blocked 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdf9e7fc640 20 dequeued request req=0x558b3952af20 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdf9e7fc640 20 RGWWQ: empty 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdf9e7fc640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 : miss 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdf9e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdf9e7fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdf9e7fc640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 info.flags=0x17 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdf9e7fc640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 to cache LRU end 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdf9e7fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdf9e7fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdf9e7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdf9e7fc640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-50 : miss 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.655+0000 7fdf9e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdf9e7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdf9e7fc640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-50 info.flags=0x0 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdf9e7fc640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-50 to cache LRU end 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdf9e7fc640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-50 : miss 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdf9e7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdf9e7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdf9e7fc640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-50 info.flags=0x0 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdf9e7fc640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-50 to cache LRU end 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdf9e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10): adding source pipe: {s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdf9e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10): adding dest pipe: {s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdf9e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdf9e7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdf9e7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39541ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39541ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39541ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39541ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39536920:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b3953f300:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b3953f300:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.655 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = cfa9b70cab31838c429301e6f493ec7d1c2b3d58414a65a28aea43cf8fe799ff 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:cfa9b70cab31838c429301e6f493ec7d1c2b3d58414a65a28aea43cf8fe799ff 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = a18ddaee51c49bfac649d192c9fe620179446e809b8ff2321ee5be075811e5fa 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a18ddaee51c49bfac649d192c9fe620179446e809b8ff2321ee5be075811e5fa 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-50%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=31, curl_handle=0x558b39537db0 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdfe23ce900 20 run: stack=0x558b39525130 is io blocked 2026-03-20T11:47:08.656 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.656+0000 7fdf88fe1640 20 link_request req_data=0x7fdf800062a0 req_data->id=31, curl_handle=0x558b39537db0 2026-03-20T11:47:08.656 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.656+0000 7f770ed0f640 -1 req 13112118919974194871 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.682 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596e40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.682 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596e40:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.682 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.682 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdfe23ce900 15 stack 0x558b3957acd0 end 2026-03-20T11:47:08.682 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdfe23ce900 20 run: stack=0x558b3957acd0 is done 2026-03-20T11:47:08.682 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.682 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdfe23ce900 20 collect(): s=0x558b3943e140 stack=0x558b3957acd0 is complete 2026-03-20T11:47:08.682 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.682 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.682 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.682 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.683 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdfe23ce900 15 stack 0x558b3943e140 end 2026-03-20T11:47:08.683 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.683 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is done 2026-03-20T11:47:08.683 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.683 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.683 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.683 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.683 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:47:08.683 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.683 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.683 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdfe23ce900 20 enqueued request req=0x558b390d20e0 2026-03-20T11:47:08.683 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.683 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdfe23ce900 20 req: 0x558b390d20e0 2026-03-20T11:47:08.683 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is io blocked 2026-03-20T11:47:08.683 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdf9dffb640 20 dequeued request req=0x558b390d20e0 2026-03-20T11:47:08.683 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdf9dffb640 20 RGWWQ: empty 2026-03-20T11:47:08.683 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdf9dffb640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 : miss 2026-03-20T11:47:08.683 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.684+0000 7fdf9dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.683 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.685+0000 7fdf9dffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.683 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.685+0000 7fdf9dffb640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 info.flags=0x17 2026-03-20T11:47:08.683 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.685+0000 7fdf9dffb640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 to cache LRU end 2026-03-20T11:47:08.683 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.685+0000 7fdf9dffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.683 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.685+0000 7fdf9dffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.683 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.685+0000 7fdf9dffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:47:08.683 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.685+0000 7fdf9dffb640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-24 : miss 2026-03-20T11:47:08.683 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.685+0000 7fdf9dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.684 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdf9dffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.684 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdf9dffb640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-24 info.flags=0x0 2026-03-20T11:47:08.684 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdf9dffb640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-24 to cache LRU end 2026-03-20T11:47:08.684 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdf9dffb640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-24 : miss 2026-03-20T11:47:08.684 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdf9dffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.684 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdf9dffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.684 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdf9dffb640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-24 info.flags=0x0 2026-03-20T11:47:08.684 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdf9dffb640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-24 to cache LRU end 2026-03-20T11:47:08.684 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdf9dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29): adding source pipe: {s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.684 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdf9dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29): adding dest pipe: {s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.684 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdf9dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.684 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdf9dffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.684 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdf9dffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29 2026-03-20T11:47:08.684 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.684 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.684 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.684 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.684 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.684 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596e40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.684 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.684 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.684 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.684 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.684 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.684 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.684 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.684 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.684 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 4e7adf99b91c80f22a8de6608d3aaaeb82beee22c448715a78d4c02129e95b11 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:4e7adf99b91c80f22a8de6608d3aaaeb82beee22c448715a78d4c02129e95b11 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 5ded97430c75fa0bf6d41f6bc7d39d98be407b7d1449848d134470a4a92c6113 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5ded97430c75fa0bf6d41f6bc7d39d98be407b7d1449848d134470a4a92c6113 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-24%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=32, curl_handle=0x558b3953c0d0 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdfe23ce900 20 run: stack=0x558b3957acd0 is io blocked 2026-03-20T11:47:08.685 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.686+0000 7fdf88fe1640 20 link_request req_data=0x558b3953c7e0 req_data->id=32, curl_handle=0x558b3953c0d0 2026-03-20T11:47:08.685 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.687+0000 7f76e34b8640 -1 req 4476148528798480159 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3952b1e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3952b1e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3959b8a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdfe23ce900 15 stack 0x558b39540cc0 end 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdfe23ce900 20 run: stack=0x558b39540cc0 is done 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdfe23ce900 20 collect(): s=0x558b3952bf80 stack=0x558b39540cc0 is complete 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdfe23ce900 15 stack 0x558b3952bf80 end 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is done 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdfe23ce900 20 enqueued request req=0x558b395473f0 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdfe23ce900 20 req: 0x558b395473f0 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is io blocked 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdf97fff640 20 dequeued request req=0x558b395473f0 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdf97fff640 20 RGWWQ: empty 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdf97fff640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 : miss 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdf97fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdf97fff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdf97fff640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 info.flags=0x17 2026-03-20T11:47:08.686 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdf97fff640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 to cache LRU end 2026-03-20T11:47:08.687 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdf97fff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.687 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdf97fff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.687 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdf97fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:47:08.687 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdf97fff640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-56 : miss 2026-03-20T11:47:08.687 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.687+0000 7fdf97fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.687 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdf97fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.687 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdf97fff640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-56 info.flags=0x0 2026-03-20T11:47:08.687 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdf97fff640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-56 to cache LRU end 2026-03-20T11:47:08.687 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdf97fff640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-56 : miss 2026-03-20T11:47:08.687 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdf97fff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.687 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdf97fff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.687 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.688+0000 7f7708502640 -1 req 10459752917268921863 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdf97fff640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-56 info.flags=0x0 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdf97fff640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-56 to cache LRU end 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdf97fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16): adding source pipe: {s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdf97fff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16): adding dest pipe: {s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdf97fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdf97fff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdf97fff640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3957aea0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3952b1e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957aea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957aea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = e7c9555013ee3d8f95ea79c8ad13d3ee6ef86f405abdf405fb58bc89a1837647 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:e7c9555013ee3d8f95ea79c8ad13d3ee6ef86f405abdf405fb58bc89a1837647 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 20dd42852da332362722dd5064ecff6534fb5a8ab6407ca7be1ece1bd472aed5 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=20dd42852da332362722dd5064ecff6534fb5a8ab6407ca7be1ece1bd472aed5 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=33, curl_handle=0x558b3952f650 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdfe23ce900 20 run: stack=0x558b39540cc0 is io blocked 2026-03-20T11:47:08.688 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.688+0000 7fdf88fe1640 20 link_request req_data=0x558b39530150 req_data->id=33, curl_handle=0x558b3952f650 2026-03-20T11:47:08.694 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39532dd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.694 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39532dd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.694 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39540e00:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.694 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdfe23ce900 15 stack 0x558b39357a70 end 2026-03-20T11:47:08.694 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdfe23ce900 20 run: stack=0x558b39357a70 is done 2026-03-20T11:47:08.694 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.694 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdfe23ce900 20 collect(): s=0x558b3943dd50 stack=0x558b39357a70 is complete 2026-03-20T11:47:08.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdfe23ce900 15 stack 0x558b3943dd50 end 2026-03-20T11:47:08.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is done 2026-03-20T11:47:08.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:47:08.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b390cf970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b390cf970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdfe23ce900 20 enqueued request req=0x558b39534d90 2026-03-20T11:47:08.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdfe23ce900 20 req: 0x558b39534d90 2026-03-20T11:47:08.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is io blocked 2026-03-20T11:47:08.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdf9cff9640 20 dequeued request req=0x558b39534d90 2026-03-20T11:47:08.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdf9cff9640 20 RGWWQ: empty 2026-03-20T11:47:08.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdf9cff9640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 : miss 2026-03-20T11:47:08.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdf9cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdf9cff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdf9cff9640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 info.flags=0x17 2026-03-20T11:47:08.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdf9cff9640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 to cache LRU end 2026-03-20T11:47:08.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdf9cff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.695 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdf9cff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdf9cff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:47:08.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdf9cff9640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-27 : miss 2026-03-20T11:47:08.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdf9cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdf9cff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdf9cff9640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-27 info.flags=0x0 2026-03-20T11:47:08.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdf9cff9640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-27 to cache LRU end 2026-03-20T11:47:08.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdf9cff9640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-27 : miss 2026-03-20T11:47:08.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.696+0000 7fdf9cff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b3953f300:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b3953f300:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdf9cff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdf9cff9640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-27 info.flags=0x0 2026-03-20T11:47:08.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39536920:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdf9cff9640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-27 to cache LRU end 2026-03-20T11:47:08.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 15 stack 0x558b39525130 end 2026-03-20T11:47:08.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 run: stack=0x558b39525130 is done 2026-03-20T11:47:08.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.696 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 collect(): s=0x558b395380a0 stack=0x558b39525130 is complete 2026-03-20T11:47:08.696 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.697+0000 7f7722536640 -1 req 3278237009421660448 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdf9cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32): adding source pipe: {s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 15 stack 0x558b395380a0 end 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is done 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdf9cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32): adding dest pipe: {s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39541ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39541ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdf9cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 enqueued request req=0x558b3952af20 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 req: 0x558b3952af20 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdf9cff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is io blocked 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdf9cff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdf977fe640 20 dequeued request req=0x558b3952af20 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdf977fe640 20 RGWWQ: empty 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdf977fe640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 : miss 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdf977fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b390cf970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b390cf970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b390cf970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b390cf970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39536920:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.697 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.698 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.698 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b3953f300:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.698 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b3953f300:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.698 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.698 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.698 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.698 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.698 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.698 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.698 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.698 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.698 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.698 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.698 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.698 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.698 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.698 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.698 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.698 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.698 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.698 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.698 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.698 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 6171febd664523322cdc7e84b9d1ab9efb8ed020f0a681b6161e2f7f862ab1e1 2026-03-20T11:47:08.698 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.699+0000 7f76e5cbd640 -1 req 17911984905508409235 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.699 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.699 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.699 INFO:teuthology.orchestra.run.vm07.stderr:6171febd664523322cdc7e84b9d1ab9efb8ed020f0a681b6161e2f7f862ab1e1 2026-03-20T11:47:08.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = c4bfe0d2347490ca1a2be3536512753dcc529b830a6d6cc29352360f7fa7bea0 2026-03-20T11:47:08.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c4bfe0d2347490ca1a2be3536512753dcc529b830a6d6cc29352360f7fa7bea0 2026-03-20T11:47:08.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=34, curl_handle=0x558b39537db0 2026-03-20T11:47:08.699 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdfe23ce900 20 run: stack=0x558b39525130 is io blocked 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.697+0000 7fdf88fe1640 20 link_request req_data=0x7fdf800062a0 req_data->id=34, curl_handle=0x558b39537db0 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdf977fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdf977fe640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 info.flags=0x17 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdf977fe640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 to cache LRU end 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdf977fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdf977fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdf977fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdf977fe640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-44 : miss 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdf977fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdf977fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdf977fe640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-44 info.flags=0x0 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdf977fe640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-44 to cache LRU end 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdf977fe640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-44 : miss 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdf977fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdf977fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdf977fe640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-44 info.flags=0x0 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdf977fe640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-44 to cache LRU end 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdf977fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4): adding source pipe: {s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdf977fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4): adding dest pipe: {s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdf977fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdf977fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdf977fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39541ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39541ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39541ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39541ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3959b8a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39532dd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39532dd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.700 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.701 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.701 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.701 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.701 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.701 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.701 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.701 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = c865e3a16a6219b39e33d4490c15ad988706640eaef489cc1746920e96a5f6a0 2026-03-20T11:47:08.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.701 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.701 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.701 INFO:teuthology.orchestra.run.vm07.stderr:c865e3a16a6219b39e33d4490c15ad988706640eaef489cc1746920e96a5f6a0 2026-03-20T11:47:08.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 368ad157d5c12c56de8f179e29f5f9923ce7d9630f411ed3c0ca85545a13452e 2026-03-20T11:47:08.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=368ad157d5c12c56de8f179e29f5f9923ce7d9630f411ed3c0ca85545a13452e 2026-03-20T11:47:08.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=35, curl_handle=0x558b39534860 2026-03-20T11:47:08.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdfe23ce900 20 run: stack=0x558b39357a70 is io blocked 2026-03-20T11:47:08.701 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.698+0000 7fdf88fe1640 20 link_request req_data=0x7fdf7c005e00 req_data->id=35, curl_handle=0x558b39534860 2026-03-20T11:47:08.728 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.728 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.728 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596e40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.728 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdfe23ce900 15 stack 0x558b3957acd0 end 2026-03-20T11:47:08.728 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdfe23ce900 20 run: stack=0x558b3957acd0 is done 2026-03-20T11:47:08.728 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.728 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdfe23ce900 20 collect(): s=0x558b3943e140 stack=0x558b3957acd0 is complete 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdfe23ce900 15 stack 0x558b3943e140 end 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is done 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdfe23ce900 20 enqueued request req=0x558b390d20e0 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdfe23ce900 20 req: 0x558b390d20e0 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is io blocked 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdf96ffd640 20 dequeued request req=0x558b390d20e0 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdf96ffd640 20 RGWWQ: empty 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdf96ffd640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 : miss 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdf96ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdf96ffd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdf96ffd640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 info.flags=0x17 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdf96ffd640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 to cache LRU end 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdf96ffd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdf96ffd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdf96ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdf96ffd640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-17 : miss 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.728+0000 7fdf96ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957aea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957aea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3952b1e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 15 stack 0x558b39540cc0 end 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 run: stack=0x558b39540cc0 is done 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.729 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 collect(): s=0x558b3952bf80 stack=0x558b39540cc0 is complete 2026-03-20T11:47:08.730 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.730 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.730 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.730 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.730 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 15 stack 0x558b3952bf80 end 2026-03-20T11:47:08.730 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.730 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is done 2026-03-20T11:47:08.730 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.730 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.730 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.730 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf96ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.730 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.730 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf96ffd640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-17 info.flags=0x0 2026-03-20T11:47:08.730 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:47:08.730 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b39580a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.730 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b39580a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.730 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 enqueued request req=0x558b395473f0 2026-03-20T11:47:08.730 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.730 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 req: 0x558b395473f0 2026-03-20T11:47:08.730 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is io blocked 2026-03-20T11:47:08.730 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf96ffd640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-17 to cache LRU end 2026-03-20T11:47:08.730 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf96ffd640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-17 : miss 2026-03-20T11:47:08.730 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf96ffd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.730 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf9d7fa640 20 dequeued request req=0x558b395473f0 2026-03-20T11:47:08.730 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf9d7fa640 20 RGWWQ: empty 2026-03-20T11:47:08.730 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf9d7fa640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 : miss 2026-03-20T11:47:08.730 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.730+0000 7f7717520640 -1 req 13008960563849851734 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.731 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf9d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.731 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf9d7fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:47:08.731 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf9d7fa640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 info.flags=0x17 2026-03-20T11:47:08.731 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf9d7fa640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 to cache LRU end 2026-03-20T11:47:08.731 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf9d7fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.731 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf9d7fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.731 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf9d7fa640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:47:08.731 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf9d7fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:47:08.731 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf9d7fa640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-46 : miss 2026-03-20T11:47:08.731 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf9d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.731 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf96ffd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.731 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf96ffd640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-17 info.flags=0x0 2026-03-20T11:47:08.731 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf96ffd640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-17 to cache LRU end 2026-03-20T11:47:08.731 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf96ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22): adding source pipe: {s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.731 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf96ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22): adding dest pipe: {s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.731 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf96ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.731 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf96ffd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.731 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf96ffd640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:47:08.731 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.731 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.731 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.731 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.731 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.731 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b39596e40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.732 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.731+0000 7f7727540640 -1 req 13511340967916709095 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3952b1e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3952b1e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf9d7fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf9d7fa640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-46 info.flags=0x0 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf9d7fa640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-46 to cache LRU end 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf9d7fa640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-46 : miss 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf9d7fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 1b0d8ca28b4cb8d5b51f2b84c70adb99856d643e2816f71aaf45f929a80d333c 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:1b0d8ca28b4cb8d5b51f2b84c70adb99856d643e2816f71aaf45f929a80d333c 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.733 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 760e746b4d60129cce18a8c0a6884c9b2b38baf10ba232db44ea56f3df988508 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=760e746b4d60129cce18a8c0a6884c9b2b38baf10ba232db44ea56f3df988508 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=36, curl_handle=0x558b3952f650 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdfe23ce900 20 run: stack=0x558b39540cc0 is io blocked 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.729+0000 7fdf88fe1640 20 link_request req_data=0x558b39530150 req_data->id=36, curl_handle=0x558b3952f650 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdf9d7fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdf9d7fa640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-46 info.flags=0x0 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdf9d7fa640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-46 to cache LRU end 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdf9d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6): adding source pipe: {s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdf9d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6): adding dest pipe: {s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdf9d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdf9d7fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdf9d7fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b39580a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b39580a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b39580a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b39580a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3957aea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3957aea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = d7c91fc38d50bff888447b37f6f1793432d971e65b2b8fb21226c7a8c2839b8f 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:d7c91fc38d50bff888447b37f6f1793432d971e65b2b8fb21226c7a8c2839b8f 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.734 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 893b1a98517ff19a92f5b1c8d8ce9ae9a353a0155a8c56874fa71fd978de5e3b 2026-03-20T11:47:08.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=893b1a98517ff19a92f5b1c8d8ce9ae9a353a0155a8c56874fa71fd978de5e3b 2026-03-20T11:47:08.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=37, curl_handle=0x558b3953c0d0 2026-03-20T11:47:08.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdfe23ce900 20 run: stack=0x558b3957acd0 is io blocked 2026-03-20T11:47:08.735 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.730+0000 7fdf88fe1640 20 link_request req_data=0x558b3953c7e0 req_data->id=37, curl_handle=0x558b3953c0d0 2026-03-20T11:47:08.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b3953f300:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b3953f300:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39536920:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdfe23ce900 15 stack 0x558b39525130 end 2026-03-20T11:47:08.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdfe23ce900 20 run: stack=0x558b39525130 is done 2026-03-20T11:47:08.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdfe23ce900 20 collect(): s=0x558b3943dd50 stack=0x558b39525130 is complete 2026-03-20T11:47:08.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdfe23ce900 15 stack 0x558b3943dd50 end 2026-03-20T11:47:08.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is done 2026-03-20T11:47:08.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.736 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:47:08.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39580a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39580a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdfe23ce900 20 enqueued request req=0x558b39534d90 2026-03-20T11:47:08.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdfe23ce900 20 req: 0x558b39534d90 2026-03-20T11:47:08.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is io blocked 2026-03-20T11:47:08.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdf967fc640 20 dequeued request req=0x558b39534d90 2026-03-20T11:47:08.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdf967fc640 20 RGWWQ: empty 2026-03-20T11:47:08.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdf967fc640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 : miss 2026-03-20T11:47:08.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdf967fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdf967fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdf967fc640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 info.flags=0x17 2026-03-20T11:47:08.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdf967fc640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 to cache LRU end 2026-03-20T11:47:08.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdf967fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdf967fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdf967fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:47:08.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdf967fc640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-37 : miss 2026-03-20T11:47:08.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdf967fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdf967fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdf967fc640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-37 info.flags=0x0 2026-03-20T11:47:08.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdf967fc640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-37 to cache LRU end 2026-03-20T11:47:08.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdf967fc640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-37 : miss 2026-03-20T11:47:08.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.738+0000 7fdf967fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.737 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdf967fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdf967fc640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-37 info.flags=0x0 2026-03-20T11:47:08.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdf967fc640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-37 to cache LRU end 2026-03-20T11:47:08.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdf967fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42): adding source pipe: {s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdf967fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42): adding dest pipe: {s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdf967fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdf967fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdf967fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:47:08.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39580a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39580a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39580a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39580a40:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.738 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.738 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.740+0000 7f7639b65640 -1 req 2301789940339759321 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b3953f300:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39580a40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39580a40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = a76f26618b29f773c71d815f30915e30e4fe32fe0b85da870d0ce19fff17b178 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:a76f26618b29f773c71d815f30915e30e4fe32fe0b85da870d0ce19fff17b178 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = ee9b22754669d2af3d9e34f313e73069c640120dd697b04de750e5b15133315e 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=ee9b22754669d2af3d9e34f313e73069c640120dd697b04de750e5b15133315e 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.739 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.740 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.740 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=38, curl_handle=0x558b39537db0 2026-03-20T11:47:08.740 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdfe23ce900 20 run: stack=0x558b39525130 is io blocked 2026-03-20T11:47:08.740 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.739+0000 7fdf88fe1640 20 link_request req_data=0x7fdf800062a0 req_data->id=38, curl_handle=0x558b39537db0 2026-03-20T11:47:08.740 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39532dd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.740 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39532dd0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.740 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3959b8a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.740 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdfe23ce900 15 stack 0x558b39357a70 end 2026-03-20T11:47:08.740 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdfe23ce900 20 run: stack=0x558b39357a70 is done 2026-03-20T11:47:08.740 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.740 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdfe23ce900 20 collect(): s=0x558b395380a0 stack=0x558b39357a70 is complete 2026-03-20T11:47:08.740 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.740 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.740 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.740 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.740 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdfe23ce900 15 stack 0x558b395380a0 end 2026-03-20T11:47:08.740 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.740 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is done 2026-03-20T11:47:08.740 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.740 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.740 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.740 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.740 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:47:08.740 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39541ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.740 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39541ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.740 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.741+0000 7f771ad27640 -1 req 3971359172540820153 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdfe23ce900 20 enqueued request req=0x558b3952af20 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdfe23ce900 20 req: 0x558b3952af20 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is io blocked 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdf95ffb640 20 dequeued request req=0x558b3952af20 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdf95ffb640 20 RGWWQ: empty 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdf95ffb640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 : miss 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdf95ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdf95ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdf95ffb640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 info.flags=0x17 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdf95ffb640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 to cache LRU end 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdf95ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdf95ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdf95ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdf95ffb640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-1 : miss 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdf95ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdf95ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdf95ffb640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-1 info.flags=0x0 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdf95ffb640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-1 to cache LRU end 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdf95ffb640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-1 : miss 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.740+0000 7fdf95ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdf95ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdf95ffb640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-1 info.flags=0x0 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdf95ffb640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-1 to cache LRU end 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdf95ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1): adding source pipe: {s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdf95ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1): adding dest pipe: {s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdf95ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdf95ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdf95ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39541ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39541ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39541ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39541ad0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39532dd0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39541ad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39541ad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.741 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.742 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.742 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.742 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.742 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.742 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.742 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = de1ba26df7a5f4bc88f47c35e83b061862b3689bbcf7497d2dce3fb945104f19 2026-03-20T11:47:08.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.742 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.742 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.742 INFO:teuthology.orchestra.run.vm07.stderr:de1ba26df7a5f4bc88f47c35e83b061862b3689bbcf7497d2dce3fb945104f19 2026-03-20T11:47:08.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 2edfd40b0c2aaf33a5b03b01db9ea58f0c6d953f0a87134feaa009f9471ddc09 2026-03-20T11:47:08.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2edfd40b0c2aaf33a5b03b01db9ea58f0c6d953f0a87134feaa009f9471ddc09 2026-03-20T11:47:08.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=39, curl_handle=0x558b39534860 2026-03-20T11:47:08.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdf88fe1640 20 link_request req_data=0x7fdf7c005e00 req_data->id=39, curl_handle=0x558b39534860 2026-03-20T11:47:08.742 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.741+0000 7fdfe23ce900 20 run: stack=0x558b39357a70 is io blocked 2026-03-20T11:47:08.769 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3952b1e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.769 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3952b1e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.769 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b39596e40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.769 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdfe23ce900 15 stack 0x558b39540cc0 end 2026-03-20T11:47:08.769 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdfe23ce900 20 run: stack=0x558b39540cc0 is done 2026-03-20T11:47:08.769 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.769 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdfe23ce900 20 collect(): s=0x558b3943e140 stack=0x558b39540cc0 is complete 2026-03-20T11:47:08.769 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.769 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.769 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.769 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.769 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdfe23ce900 15 stack 0x558b3943e140 end 2026-03-20T11:47:08.769 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.769 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is done 2026-03-20T11:47:08.769 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.770 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.770 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.770 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.770 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:47:08.770 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.770 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.770 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdfe23ce900 20 enqueued request req=0x558b390d20e0 2026-03-20T11:47:08.770 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.770 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdfe23ce900 20 req: 0x558b390d20e0 2026-03-20T11:47:08.770 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is io blocked 2026-03-20T11:47:08.770 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdf957fa640 20 dequeued request req=0x558b390d20e0 2026-03-20T11:47:08.770 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdf957fa640 20 RGWWQ: empty 2026-03-20T11:47:08.770 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdf957fa640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 : miss 2026-03-20T11:47:08.770 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdf957fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.770 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdf957fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:47:08.770 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdf957fa640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 info.flags=0x17 2026-03-20T11:47:08.770 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdf957fa640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 to cache LRU end 2026-03-20T11:47:08.770 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdf957fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.770 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdf957fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.770 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdf957fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:47:08.770 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdf957fa640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-7 : miss 2026-03-20T11:47:08.770 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdf957fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.770 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdf957fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdf957fa640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-7 info.flags=0x0 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdf957fa640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-7 to cache LRU end 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdf957fa640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-7 : miss 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.771+0000 7fdf957fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3957aea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3957aea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 15 stack 0x558b3957acd0 end 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 run: stack=0x558b3957acd0 is done 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 collect(): s=0x558b3952bf80 stack=0x558b3957acd0 is complete 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 15 stack 0x558b3952bf80 end 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is done 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b390cada0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b390cada0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 enqueued request req=0x558b395473f0 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 req: 0x558b395473f0 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is io blocked 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdf94ff9640 20 dequeued request req=0x558b395473f0 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdf94ff9640 20 RGWWQ: empty 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdf94ff9640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 : miss 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdf94ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdf957fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdf957fa640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-7 info.flags=0x0 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdf957fa640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-7 to cache LRU end 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdf957fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12): adding source pipe: {s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdf957fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12): adding dest pipe: {s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdf957fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdf957fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdf957fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.771 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596e40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 90b79b1d02a932ecde42e604079392a82711cc7f565bdf200acdc0f543b70a2f 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:90b79b1d02a932ecde42e604079392a82711cc7f565bdf200acdc0f543b70a2f 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = cfd3b80b7fd8a00922278753022d40030e96364b104b5a79a3b6b7510e5f2362 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=cfd3b80b7fd8a00922278753022d40030e96364b104b5a79a3b6b7510e5f2362 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=40, curl_handle=0x558b3953c0d0 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdf88fe1640 20 link_request req_data=0x558b3953c7e0 req_data->id=40, curl_handle=0x558b3953c0d0 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdfe23ce900 20 run: stack=0x558b3957acd0 is io blocked 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdf94ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:47:08.772 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdf94ff9640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 info.flags=0x17 2026-03-20T11:47:08.772 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.772+0000 7f770fd11640 -1 req 8024147638781064058 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdf94ff9640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 to cache LRU end 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdf94ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdf94ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdf94ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdf94ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdf94ff9640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-48 : miss 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdf94ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdf94ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdf94ff9640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-48 info.flags=0x0 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdf94ff9640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-48 to cache LRU end 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdf94ff9640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-48 : miss 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.772+0000 7fdf94ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdf94ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdf94ff9640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-48 info.flags=0x0 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdf94ff9640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-48 to cache LRU end 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdf94ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8): adding source pipe: {s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdf94ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8): adding dest pipe: {s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdf94ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdf94ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdf94ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b390cada0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b390cada0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b390cada0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b390cada0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3959b8a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957aea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957aea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 79e056b9b9f489b97d7180c4b387eed8d30f101608a3ea2212af9adeb860e0ae 2026-03-20T11:47:08.773 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.774 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.774 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.774 INFO:teuthology.orchestra.run.vm07.stderr:79e056b9b9f489b97d7180c4b387eed8d30f101608a3ea2212af9adeb860e0ae 2026-03-20T11:47:08.774 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.774 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.774 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.774 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.774 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = af3362ba390efe3c82250d43994ce99d67c670044442c89038401f0ef9a677a3 2026-03-20T11:47:08.774 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=af3362ba390efe3c82250d43994ce99d67c670044442c89038401f0ef9a677a3 2026-03-20T11:47:08.774 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.774 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.774 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.774 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=41, curl_handle=0x558b3952f650 2026-03-20T11:47:08.774 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdf88fe1640 20 link_request req_data=0x558b39530150 req_data->id=41, curl_handle=0x558b3952f650 2026-03-20T11:47:08.774 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.773+0000 7f772453a640 -1 req 5407773982042839725 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.774 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.773+0000 7fdfe23ce900 20 run: stack=0x558b39540cc0 is io blocked 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39580a40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39580a40:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b3953f300:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdfe23ce900 15 stack 0x558b39525130 end 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdfe23ce900 20 run: stack=0x558b39525130 is done 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdfe23ce900 20 collect(): s=0x558b3943dd50 stack=0x558b39525130 is complete 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdfe23ce900 15 stack 0x558b3943dd50 end 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is done 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b390cf970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b390cf970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdfe23ce900 20 enqueued request req=0x558b39534d90 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdfe23ce900 20 req: 0x558b39534d90 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is io blocked 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdf947f8640 20 dequeued request req=0x558b39534d90 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdf947f8640 20 RGWWQ: empty 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdf947f8640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 : miss 2026-03-20T11:47:08.779 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdf947f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdf947f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdf947f8640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 info.flags=0x17 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdf947f8640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 to cache LRU end 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdf947f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdf947f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdf947f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdf947f8640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-59 : miss 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdf947f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdf947f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdf947f8640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-59 info.flags=0x0 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdf947f8640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-59 to cache LRU end 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdf947f8640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-59 : miss 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdf947f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdf947f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdf947f8640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-59 info.flags=0x0 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdf947f8640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-59 to cache LRU end 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdf947f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19): adding source pipe: {s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdf947f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19): adding dest pipe: {s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdf947f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.781+0000 7fdf947f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdf947f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b390cf970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b390cf970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b390cf970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b390cf970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b3953f300:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39580a40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39580a40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.780 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 836771e3af7ffecf51c9f984d7886893c51640e9b44dabb2a38ca4db30e24d12 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:836771e3af7ffecf51c9f984d7886893c51640e9b44dabb2a38ca4db30e24d12 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 0efbdfe3e1c1475f8812901b2b7a3f0b1221caf25b09e4423b19201e42ade678 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=0efbdfe3e1c1475f8812901b2b7a3f0b1221caf25b09e4423b19201e42ade678 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=42, curl_handle=0x558b39537db0 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdf88fe1640 20 link_request req_data=0x7fdf800062a0 req_data->id=42, curl_handle=0x558b39537db0 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.782+0000 7fdfe23ce900 20 run: stack=0x558b39525130 is io blocked 2026-03-20T11:47:08.781 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.782+0000 7f765fbb1640 -1 req 3650052592156294482 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39541ad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39541ad0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39532dd0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdfe23ce900 15 stack 0x558b39357a70 end 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdfe23ce900 20 run: stack=0x558b39357a70 is done 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdfe23ce900 20 collect(): s=0x558b395380a0 stack=0x558b39357a70 is complete 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.781 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdfe23ce900 15 stack 0x558b395380a0 end 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is done 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b3953ffe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b3953ffe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdfe23ce900 20 enqueued request req=0x558b3952af20 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdfe23ce900 20 req: 0x558b3952af20 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is io blocked 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdf93ff7640 20 dequeued request req=0x558b3952af20 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdf93ff7640 20 RGWWQ: empty 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdf93ff7640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 : miss 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdf93ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdf93ff7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdf93ff7640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 info.flags=0x17 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdf93ff7640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 to cache LRU end 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdf93ff7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdf93ff7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdf93ff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdf93ff7640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-6 : miss 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdf93ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdf93ff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdf93ff7640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-6 info.flags=0x0 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdf93ff7640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-6 to cache LRU end 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdf93ff7640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-6 : miss 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdf93ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdf93ff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdf93ff7640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-6 info.flags=0x0 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdf93ff7640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-6 to cache LRU end 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdf93ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11): adding source pipe: {s={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.783+0000 7fdf93ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11): adding dest pipe: {s={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.782 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdf93ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdf93ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdf93ff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b3953ffe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b3953ffe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b3953ffe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b3953ffe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.783 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.784+0000 7f7728d43640 -1 req 6049604297291704935 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-6:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39541ad0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3952b1e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3952b1e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 22235c3ef48d72abb9800dddf769df3c86ed0ba3050990c904e138a3f5062449 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:22235c3ef48d72abb9800dddf769df3c86ed0ba3050990c904e138a3f5062449 2026-03-20T11:47:08.783 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 44c4bb7aba6b81452b48e5e34800a664c5b57d89e60803bbc8c49ead53697d66 2026-03-20T11:47:08.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=44c4bb7aba6b81452b48e5e34800a664c5b57d89e60803bbc8c49ead53697d66 2026-03-20T11:47:08.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-6%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.11&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=43, curl_handle=0x558b39534860 2026-03-20T11:47:08.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdf88fe1640 20 link_request req_data=0x7fdf7c005e00 req_data->id=43, curl_handle=0x558b39534860 2026-03-20T11:47:08.784 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.784+0000 7fdfe23ce900 20 run: stack=0x558b39357a70 is io blocked 2026-03-20T11:47:08.811 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.811 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.811 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b39596e40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.811 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdfe23ce900 15 stack 0x558b3957acd0 end 2026-03-20T11:47:08.811 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdfe23ce900 20 run: stack=0x558b3957acd0 is done 2026-03-20T11:47:08.811 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.811 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdfe23ce900 20 collect(): s=0x558b3943e140 stack=0x558b3957acd0 is complete 2026-03-20T11:47:08.811 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.811 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.811 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.811 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.811 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdfe23ce900 15 stack 0x558b3943e140 end 2026-03-20T11:47:08.811 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.812 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is done 2026-03-20T11:47:08.812 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.812 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.812 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.812 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.812 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:47:08.812 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.812 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.812 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdfe23ce900 20 enqueued request req=0x558b390d20e0 2026-03-20T11:47:08.812 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.812 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdfe23ce900 20 req: 0x558b390d20e0 2026-03-20T11:47:08.812 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is io blocked 2026-03-20T11:47:08.812 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdf937f6640 20 dequeued request req=0x558b390d20e0 2026-03-20T11:47:08.812 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdf937f6640 20 RGWWQ: empty 2026-03-20T11:47:08.812 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdf937f6640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 : miss 2026-03-20T11:47:08.812 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdf937f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.813 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdf937f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.813 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdf937f6640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 info.flags=0x17 2026-03-20T11:47:08.813 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdf937f6640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 to cache LRU end 2026-03-20T11:47:08.813 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdf937f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.813 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdf937f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.813 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdf937f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:47:08.813 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdf937f6640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-21 : miss 2026-03-20T11:47:08.813 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.813+0000 7fdf937f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.813 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957aea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.813 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957aea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.813 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3959b8a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.813 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 15 stack 0x558b39540cc0 end 2026-03-20T11:47:08.813 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 run: stack=0x558b39540cc0 is done 2026-03-20T11:47:08.813 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.813 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 collect(): s=0x558b3952bf80 stack=0x558b39540cc0 is complete 2026-03-20T11:47:08.813 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.813 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf937f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.813 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.813 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.813 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.813 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 15 stack 0x558b3952bf80 end 2026-03-20T11:47:08.813 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf937f6640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-21 info.flags=0x0 2026-03-20T11:47:08.813 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.813 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is done 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf937f6640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-21 to cache LRU end 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf937f6640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-21 : miss 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf937f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b390cada0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b390cada0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 enqueued request req=0x558b395473f0 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 req: 0x558b395473f0 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is io blocked 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf92ff5640 20 dequeued request req=0x558b395473f0 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf92ff5640 20 RGWWQ: empty 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf92ff5640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 : miss 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf92ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf92ff5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf92ff5640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 info.flags=0x17 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf92ff5640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 to cache LRU end 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf92ff5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf92ff5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf92ff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf92ff5640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-41 : miss 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf92ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf937f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf937f6640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-21 info.flags=0x0 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf937f6640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-21 to cache LRU end 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf937f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26): adding source pipe: {s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf937f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26): adding dest pipe: {s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf937f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf937f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf937f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.814 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b39596e40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957aea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957aea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = cf0975f214efd226951b58614387975e07af2c3f38e4e9172be142cb2f72036c 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:cf0975f214efd226951b58614387975e07af2c3f38e4e9172be142cb2f72036c 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf92ff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 16a5a0ac6b9a96f7a067c12dac4bdaa2b05cfa9864a87659d0a4ce58084e19a2 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf92ff5640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-41 info.flags=0x0 2026-03-20T11:47:08.815 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=16a5a0ac6b9a96f7a067c12dac4bdaa2b05cfa9864a87659d0a4ce58084e19a2 2026-03-20T11:47:08.816 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.816 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.816 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.816 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=44, curl_handle=0x558b3952f650 2026-03-20T11:47:08.816 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf92ff5640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-41 to cache LRU end 2026-03-20T11:47:08.816 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf92ff5640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-41 : miss 2026-03-20T11:47:08.816 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf92ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.816 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdf88fe1640 20 link_request req_data=0x558b39530150 req_data->id=44, curl_handle=0x558b3952f650 2026-03-20T11:47:08.816 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.814+0000 7fdfe23ce900 20 run: stack=0x558b39540cc0 is io blocked 2026-03-20T11:47:08.816 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.815+0000 7f772653e640 -1 req 649496009243876401 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.816 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdf92ff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.816 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdf92ff5640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-41 info.flags=0x0 2026-03-20T11:47:08.816 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdf92ff5640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-41 to cache LRU end 2026-03-20T11:47:08.816 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdf92ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1): adding source pipe: {s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.816 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdf92ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1): adding dest pipe: {s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdf92ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdf92ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdf92ff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b390cada0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b390cada0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b390cada0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b390cada0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b390cada0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = d26898e8deb98dd09dc1ad27ca92f43f8d67ea8d132d4e31ae7c06b475a66ce8 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:d26898e8deb98dd09dc1ad27ca92f43f8d67ea8d132d4e31ae7c06b475a66ce8 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 92b9cb1f256f68afc6b98646fdfb432fc5057c554f26078c50a33e737517d318 2026-03-20T11:47:08.817 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=92b9cb1f256f68afc6b98646fdfb432fc5057c554f26078c50a33e737517d318 2026-03-20T11:47:08.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=45, curl_handle=0x558b3953c0d0 2026-03-20T11:47:08.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdf88fe1640 20 link_request req_data=0x558b3953c7e0 req_data->id=45, curl_handle=0x558b3953c0d0 2026-03-20T11:47:08.818 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.815+0000 7f763e36e640 -1 req 11536456360687828771 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.818 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.815+0000 7fdfe23ce900 20 run: stack=0x558b3957acd0 is io blocked 2026-03-20T11:47:08.821 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39580a40:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b39580a40:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b3953f300:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdfe23ce900 15 stack 0x558b39525130 end 2026-03-20T11:47:08.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdfe23ce900 20 run: stack=0x558b39525130 is done 2026-03-20T11:47:08.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdfe23ce900 20 collect(): s=0x558b3943dd50 stack=0x558b39525130 is complete 2026-03-20T11:47:08.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdfe23ce900 15 stack 0x558b3943dd50 end 2026-03-20T11:47:08.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is done 2026-03-20T11:47:08.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:47:08.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b390cf970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b390cf970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdfe23ce900 20 enqueued request req=0x558b39534d90 2026-03-20T11:47:08.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdfe23ce900 20 req: 0x558b39534d90 2026-03-20T11:47:08.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is io blocked 2026-03-20T11:47:08.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdf927f4640 20 dequeued request req=0x558b39534d90 2026-03-20T11:47:08.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdf927f4640 20 RGWWQ: empty 2026-03-20T11:47:08.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdf927f4640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 : miss 2026-03-20T11:47:08.822 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdf927f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdf927f4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdf927f4640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 info.flags=0x17 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdf927f4640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 to cache LRU end 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdf927f4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdf927f4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdf927f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdf927f4640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-49 : miss 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdf927f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdf927f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdf927f4640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-49 info.flags=0x0 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdf927f4640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-49 to cache LRU end 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdf927f4640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-49 : miss 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.823+0000 7fdf927f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdf927f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdf927f4640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-49 info.flags=0x0 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdf927f4640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-49 to cache LRU end 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdf927f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9): adding source pipe: {s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdf927f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9): adding dest pipe: {s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdf927f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdf927f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdf927f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b390cf970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b390cf970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b390cf970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b390cf970:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b390cf970:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b390dbfb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b390dbfb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.823 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.824 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = 9e648e8820f5603bfea2df424f9defb62190734cf2774c7e638c67431fdb972e 2026-03-20T11:47:08.824 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.824 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.824 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.824 INFO:teuthology.orchestra.run.vm07.stderr:9e648e8820f5603bfea2df424f9defb62190734cf2774c7e638c67431fdb972e 2026-03-20T11:47:08.824 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.824 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.824 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.824 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.824 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 584784cae84fac94e59251f548a88cc90ca037f9382b5606a37adea9a4ef1413 2026-03-20T11:47:08.824 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=584784cae84fac94e59251f548a88cc90ca037f9382b5606a37adea9a4ef1413 2026-03-20T11:47:08.824 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.824 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.824 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.824 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=46, curl_handle=0x558b39537db0 2026-03-20T11:47:08.824 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdf88fe1640 20 link_request req_data=0x7fdf800062a0 req_data->id=46, curl_handle=0x558b39537db0 2026-03-20T11:47:08.824 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.824+0000 7fdfe23ce900 20 run: stack=0x558b39525130 is io blocked 2026-03-20T11:47:08.824 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.824+0000 7f7665bbd640 -1 req 16999632663235790150 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.824 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3952b1e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.824 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3952b1e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.824 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39541ad0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.824 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdfe23ce900 15 stack 0x558b39357a70 end 2026-03-20T11:47:08.824 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdfe23ce900 20 run: stack=0x558b39357a70 is done 2026-03-20T11:47:08.824 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.824 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdfe23ce900 20 collect(): s=0x558b395380a0 stack=0x558b39357a70 is complete 2026-03-20T11:47:08.824 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.824 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.824 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.824 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdfe23ce900 15 stack 0x558b395380a0 end 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is done 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b3953ffe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b3953ffe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdfe23ce900 20 enqueued request req=0x558b3952af20 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdfe23ce900 20 req: 0x558b3952af20 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is io blocked 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdf91ff3640 20 dequeued request req=0x558b3952af20 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdf91ff3640 20 RGWWQ: empty 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdf91ff3640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 : miss 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdf91ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdf91ff3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdf91ff3640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 info.flags=0x17 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdf91ff3640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 to cache LRU end 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdf91ff3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdf91ff3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdf91ff3640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdf91ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdf91ff3640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-45 : miss 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdf91ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdf91ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdf91ff3640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-45 info.flags=0x0 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdf91ff3640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-45 to cache LRU end 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdf91ff3640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-45 : miss 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdf91ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdf91ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdf91ff3640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-45 info.flags=0x0 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdf91ff3640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-45 to cache LRU end 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdf91ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5): adding source pipe: {s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdf91ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5): adding dest pipe: {s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdf91ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.825+0000 7fdf91ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdf91ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b3953ffe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b3953ffe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b3953ffe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b3953ffe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39541ad0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3952b1e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3952b1e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.825 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.826 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.826 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.826 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.826 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.826 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.826 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.826 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = ac111ba731eb5cc99b7300b60b68c20d0a7e5dad3a7945dbb19c4fbc1f974ba3 2026-03-20T11:47:08.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.826 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.826 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.826 INFO:teuthology.orchestra.run.vm07.stderr:ac111ba731eb5cc99b7300b60b68c20d0a7e5dad3a7945dbb19c4fbc1f974ba3 2026-03-20T11:47:08.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = a4a92ea49520f889dfb710155c2f7fa3bdd4a3ae91060a867fed8facb3cb6371 2026-03-20T11:47:08.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a4a92ea49520f889dfb710155c2f7fa3bdd4a3ae91060a867fed8facb3cb6371 2026-03-20T11:47:08.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=47, curl_handle=0x558b39534860 2026-03-20T11:47:08.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdf88fe1640 20 link_request req_data=0x7fdf7c005e00 req_data->id=47, curl_handle=0x558b39534860 2026-03-20T11:47:08.826 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.826+0000 7f76f5cdd640 -1 req 9853513464187381554 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.826 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.826+0000 7fdfe23ce900 20 run: stack=0x558b39357a70 is io blocked 2026-03-20T11:47:08.854 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957aea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.854 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957aea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.854 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b39596e40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 15 stack 0x558b39540cc0 end 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 run: stack=0x558b39540cc0 is done 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 collect(): s=0x558b3943e140 stack=0x558b39540cc0 is complete 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 15 stack 0x558b3943e140 end 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is done 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b390cada0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 15 stack 0x558b3957acd0 end 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 run: stack=0x558b3957acd0 is done 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 collect(): s=0x558b3952bf80 stack=0x558b3957acd0 is complete 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 15 stack 0x558b3952bf80 end 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is done 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3957f710:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 4 starting trim on bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3957f710:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 enqueued request req=0x558b395336f0 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 req: 0x558b395336f0 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is io blocked 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 enqueued request req=0x558b3952d400 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 RGWWQ: 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 req: 0x558b395336f0 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 req: 0x558b3952d400 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is io blocked 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf917f2640 20 dequeued request req=0x558b395336f0 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf917f2640 20 RGWWQ: 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf917f2640 20 req: 0x558b3952d400 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf917f2640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 : miss 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf917f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf90ff1640 20 dequeued request req=0x558b3952d400 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf90ff1640 20 RGWWQ: empty 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf90ff1640 10 rgw async rados processor: cache get: name=b2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 : miss 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf90ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf917f2640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf917f2640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 info.flags=0x17 2026-03-20T11:47:08.855 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf917f2640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 to cache LRU end 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf917f2640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf917f2640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf917f2640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf917f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf917f2640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-33 : miss 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf917f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf90ff1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf90ff1640 10 rgw async rados processor: cache put: name=b2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 info.flags=0x17 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf90ff1640 10 rgw async rados processor: adding b2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 to cache LRU end 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf90ff1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf90ff1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf90ff1640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf90ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf90ff1640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-source-hints.dipepu-47 : miss 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf90ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf917f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf917f2640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-33 info.flags=0x0 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf917f2640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-33 to cache LRU end 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf917f2640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-33 : miss 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf917f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf90ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf90ff1640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-source-hints.dipepu-47 info.flags=0x0 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf90ff1640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-source-hints.dipepu-47 to cache LRU end 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf90ff1640 10 rgw async rados processor: cache get: name=b2.rgw.log++bucket.sync-target-hints.dipepu-47 : miss 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.856+0000 7fdf90ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdf917f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdf917f2640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-33 info.flags=0x0 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdf917f2640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-33 to cache LRU end 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdf917f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38): adding source pipe: {s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdf917f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38): adding dest pipe: {s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdf917f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdf917f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.856 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdf917f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3957f710:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3957f710:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3957f710:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b3957f710:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b390cada0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdf90ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdf90ff1640 10 rgw async rados processor: cache put: name=b2.rgw.log++bucket.sync-target-hints.dipepu-47 info.flags=0x0 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdf90ff1640 10 rgw async rados processor: adding b2.rgw.log++bucket.sync-target-hints.dipepu-47 to cache LRU end 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdf90ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7): adding source pipe: {s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdf90ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7): adding dest pipe: {s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdf90ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0},d={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0}} 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdf90ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=*,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = d0c479ad29794c26450948ec7e5e45591614fb353be2357a5760c0491a104005 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:d0c479ad29794c26450948ec7e5e45591614fb353be2357a5760c0491a104005 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdf90ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=b2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 6a5a82ae32f434568f112553c7f63693928872b735480b2f0f5421a9434d32be 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=6a5a82ae32f434568f112553c7f63693928872b735480b2f0f5421a9434d32be 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=48, curl_handle=0x558b3953c0d0 2026-03-20T11:47:08.857 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdf88fe1640 20 link_request req_data=0x558b3953c7e0 req_data->id=48, curl_handle=0x558b3953c0d0 2026-03-20T11:47:08.857 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.857+0000 7f7637b61640 -1 req 5730578474058423268 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 run: stack=0x558b3957acd0 is io blocked 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b3959cdd0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b39596e40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957aea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957aea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 get_url picked endpoint=http://vm07.local:8000 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4():> HTTP_HOST -> vm07.local:8000 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical headers format = date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request = GET 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:/admin/log/ 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&options=merge&rgwx-zonegroup=b&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&status=&type=bucket-index&version=2 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:date:Fri, 20 Mar 2026 11:47:08 +0000 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:host:vm07.local:8000 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:x-amz-date:20260320T114708Z 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr: 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]canonical request hash = fc15f8ca020d2f563b2d48b9f0c8738c931e9c80b221d7124edf6675d45be540 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:20260320T114708Z 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:20260320/b/s3/aws4_request 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:fc15f8ca020d2f563b2d48b9f0c8738c931e9c80b221d7124edf6675d45be540 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]region_k = 9c45e5dec06004d5b8dd112d99852bdcdf61e46d5a892028d9ff472c770d6cbc 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]service_k = 884b2362fddf390f5c8ae3cc7e1d162339f30480fca389b85d0595f0ee6852e0 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]signing_k = 2bf450d35b24bb564ef129a6b8b3e5c3582aee54badb39294aa49aba8deb64d4 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 10 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]generated signature = 292e6f37be7e3a903a6a698a54ae0394b76f89b93da1c3395b3154d6af7dbeeb 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/b/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=292e6f37be7e3a903a6a698a54ae0394b76f89b93da1c3395b3154d6af7dbeeb 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 http_client[GET/http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114708Z 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 sending request to http://vm07.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=cd52b229-0d4a-43bf-b4ba-156aa19e7aef&version=2&rgwx-zonegroup=b 2026-03-20T11:47:08.858 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 register_request mgr=0x7fff3f651fa0 req_data->id=49, curl_handle=0x558b3952f650 2026-03-20T11:47:08.859 INFO:tasks.rgw.c2.client.0.vm07.stdout:2026-03-20T11:47:08.858+0000 7f765cbab640 -1 req 2852359822927303485 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=cd52b229-0d4a-43bf-b4ba-156aa19e7aef,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=efe8d1b5-06fe-4c24-add7-5f2f1fed8c9c,az=0}} returned ret=-2 2026-03-20T11:47:08.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdf88fe1640 20 link_request req_data=0x558b39530150 req_data->id=49, curl_handle=0x558b3952f650 2026-03-20T11:47:08.859 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.857+0000 7fdfe23ce900 20 run: stack=0x558b39540cc0 is io blocked 2026-03-20T11:47:08.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.865+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b390dbfb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.865+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b390dbfb0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.865+0000 7fdfe23ce900 20 cr:s=0x558b39525130:op=0x558b390cf970:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.865+0000 7fdfe23ce900 15 stack 0x558b39525130 end 2026-03-20T11:47:08.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.865+0000 7fdfe23ce900 20 run: stack=0x558b39525130 is done 2026-03-20T11:47:08.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.865+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.865+0000 7fdfe23ce900 20 collect(): s=0x558b3943dd50 stack=0x558b39525130 is complete 2026-03-20T11:47:08.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.865+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.865+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.865+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.865+0000 7fdfe23ce900 20 cr:s=0x558b3943dd50:op=0x558b39539eb0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.865+0000 7fdfe23ce900 15 stack 0x558b3943dd50 end 2026-03-20T11:47:08.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.865+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.865+0000 7fdfe23ce900 20 run: stack=0x558b3943dd50 is done 2026-03-20T11:47:08.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.865+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.865+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.863 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.865+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.865 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.867+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3952b1e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.865 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.867+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b3952b1e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.865 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.867+0000 7fdfe23ce900 20 cr:s=0x558b39357a70:op=0x558b39541ad0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.865 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.867+0000 7fdfe23ce900 15 stack 0x558b39357a70 end 2026-03-20T11:47:08.865 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.867+0000 7fdfe23ce900 20 run: stack=0x558b39357a70 is done 2026-03-20T11:47:08.865 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.867+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.865 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.867+0000 7fdfe23ce900 20 collect(): s=0x558b395380a0 stack=0x558b39357a70 is complete 2026-03-20T11:47:08.865 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.867+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.865 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.867+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.865 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.867+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.865 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.867+0000 7fdfe23ce900 20 cr:s=0x558b395380a0:op=0x558b39523b80:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.865 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.867+0000 7fdfe23ce900 15 stack 0x558b395380a0 end 2026-03-20T11:47:08.865 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.867+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.865 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.867+0000 7fdfe23ce900 20 run: stack=0x558b395380a0 is done 2026-03-20T11:47:08.865 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.867+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.865 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.867+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.865 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.867+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.896 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.898+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.896 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.898+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b3959e140:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.896 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.898+0000 7fdfe23ce900 20 cr:s=0x558b3957acd0:op=0x558b390cada0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.896 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.898+0000 7fdfe23ce900 15 stack 0x558b3957acd0 end 2026-03-20T11:47:08.896 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.898+0000 7fdfe23ce900 20 run: stack=0x558b3957acd0 is done 2026-03-20T11:47:08.896 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.898+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.896 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.898+0000 7fdfe23ce900 20 collect(): s=0x558b3943e140 stack=0x558b3957acd0 is complete 2026-03-20T11:47:08.896 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.898+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.896 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.898+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.896 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.898+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.896 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.898+0000 7fdfe23ce900 20 cr:s=0x558b3943e140:op=0x558b39594e20:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.896 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.898+0000 7fdfe23ce900 15 stack 0x558b3943e140 end 2026-03-20T11:47:08.896 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.898+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.896 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.898+0000 7fdfe23ce900 20 run: stack=0x558b3943e140 is done 2026-03-20T11:47:08.896 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.898+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.896 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.898+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.896 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.898+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:08.897 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.899+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957aea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:08.897 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.899+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b3957aea0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:08.897 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.899+0000 7fdfe23ce900 20 cr:s=0x558b39540cc0:op=0x558b39596e40:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:08.897 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.899+0000 7fdfe23ce900 15 stack 0x558b39540cc0 end 2026-03-20T11:47:08.897 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.899+0000 7fdfe23ce900 20 run: stack=0x558b39540cc0 is done 2026-03-20T11:47:08.897 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.899+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:08.897 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.899+0000 7fdfe23ce900 20 collect(): s=0x558b3952bf80 stack=0x558b39540cc0 is complete 2026-03-20T11:47:08.897 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.899+0000 7fdfe23ce900 10 min_generation is 0 2026-03-20T11:47:08.897 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.899+0000 7fdfe23ce900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:08.897 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.899+0000 7fdfe23ce900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:08.897 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.899+0000 7fdfe23ce900 20 cr:s=0x558b3952bf80:op=0x558b395929a0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:08.897 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.899+0000 7fdfe23ce900 15 stack 0x558b3952bf80 end 2026-03-20T11:47:08.897 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.899+0000 7fdfe23ce900 20 stack->operate() returned ret=-22 2026-03-20T11:47:08.897 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.899+0000 7fdfe23ce900 20 run: stack=0x558b3952bf80 is done 2026-03-20T11:47:08.897 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.899+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:08.897 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.899+0000 7fdfe23ce900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:08.897 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.899+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39525e20:27BucketTrimInstanceCollectCR: operate() returned r=-22 2026-03-20T11:47:08.897 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.899+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39522330:12BucketTrimCR: operate() 2026-03-20T11:47:08.897 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.899+0000 7fdfe23ce900 20 writing bucket trim marker=16:a04872b5:root::dipepu-33:head 2026-03-20T11:47:08.897 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.899+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b390cf7b0:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:08.897 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.899+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b390cf7b0:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:08.897 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.899+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is io blocked 2026-03-20T11:47:08.898 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.901+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b390cf7b0:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:08.898 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.901+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b390cf7b0:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:08.898 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.901+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b390cf7b0:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:08.898 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.901+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b390cf7b0:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:08.898 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.901+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39522330:12BucketTrimCR: operate() 2026-03-20T11:47:08.898 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.901+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b390cf7b0:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:08.898 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.901+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b390cf7b0:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:08.898 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.901+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is io blocked 2026-03-20T11:47:08.898 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.901+0000 7fdfa7fff640 20 trim: bucket trim completed 2026-03-20T11:47:08.899 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.901+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b390cf7b0:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:08.899 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.901+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b390cf7b0:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:08.899 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.901+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b390cf7b0:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:08.899 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.901+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b390cf7b0:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:08.899 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.901+0000 7fdfe23ce900 20 cr:s=0x558b395220f0:op=0x558b39522330:12BucketTrimCR: operate() 2026-03-20T11:47:08.899 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.901+0000 7fdfe23ce900 4 bucket index log processing completed in 0.716499984s 2026-03-20T11:47:08.899 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.901+0000 7fdfe23ce900 15 stack 0x558b395220f0 end 2026-03-20T11:47:08.899 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.901+0000 7fdfe23ce900 20 run: stack=0x558b395220f0 is done 2026-03-20T11:47:08.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.906+0000 7fdfe23ce900 20 remove_watcher() i=5 2026-03-20T11:47:08.904 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.906+0000 7fdfe23ce900 2 removed watcher, disabling cache 2026-03-20T11:47:08.905 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.907+0000 7fdfe23ce900 20 remove_watcher() i=1 2026-03-20T11:47:08.905 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.907+0000 7fdfe23ce900 20 remove_watcher() i=2 2026-03-20T11:47:08.905 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.907+0000 7fdfe23ce900 20 remove_watcher() i=0 2026-03-20T11:47:08.905 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.907+0000 7fdfe23ce900 20 remove_watcher() i=6 2026-03-20T11:47:08.905 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.907+0000 7fdfe23ce900 20 remove_watcher() i=4 2026-03-20T11:47:08.905 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.907+0000 7fdfe23ce900 20 remove_watcher() i=7 2026-03-20T11:47:08.905 INFO:teuthology.orchestra.run.vm07.stderr:2026-03-20T11:47:08.907+0000 7fdfe23ce900 20 remove_watcher() i=3 2026-03-20T11:47:13.912 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '50', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:47:13.912 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bilog', 'autotrim', '--rgw-sync-log-trim-max-buckets', '50', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '20', '--debug-ms', '0'] 2026-03-20T11:47:13.912 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bilog autotrim --rgw-sync-log-trim-max-buckets 50 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 20 --debug-ms 0 2026-03-20T11:47:13.949 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:47:13.949 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:47:13.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.968+0000 7f48c3739900 20 rados->read ofs=0 len=0 2026-03-20T11:47:13.965 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.969+0000 7f48c3739900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:47:13.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.969+0000 7f48c3739900 20 rados->read ofs=0 len=0 2026-03-20T11:47:13.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.969+0000 7f48c3739900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:47:13.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.969+0000 7f48c3739900 20 realm test-realm 6a789b89-c2b8-45f5-82cc-fb80234d0c6c 2026-03-20T11:47:13.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.969+0000 7f48c3739900 20 rados->read ofs=0 len=0 2026-03-20T11:47:13.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.970+0000 7f48c3739900 20 rados_obj.operate() r=0 bl.length=110 2026-03-20T11:47:13.966 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.970+0000 7f48c3739900 20 rados->read ofs=0 len=0 2026-03-20T11:47:13.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.970+0000 7f48c3739900 20 rados_obj.operate() r=0 bl.length=10 2026-03-20T11:47:13.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.970+0000 7f48c3739900 20 rados->read ofs=0 len=0 2026-03-20T11:47:13.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.971+0000 7f48c3739900 20 rados_obj.operate() r=0 bl.length=2020 2026-03-20T11:47:13.967 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.971+0000 7f48c3739900 20 rados->read ofs=0 len=0 2026-03-20T11:47:13.968 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.971+0000 7f48c3739900 20 rados_obj.operate() r=0 bl.length=46 2026-03-20T11:47:13.968 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.971+0000 7f48c3739900 20 rados->read ofs=0 len=0 2026-03-20T11:47:13.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.972+0000 7f48c3739900 20 rados_obj.operate() r=0 bl.length=1030 2026-03-20T11:47:13.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.972+0000 7f48c3739900 20 zone a2 found 2026-03-20T11:47:13.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.972+0000 7f48c3739900 4 Realm: test-realm (6a789b89-c2b8-45f5-82cc-fb80234d0c6c) 2026-03-20T11:47:13.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.972+0000 7f48c3739900 4 ZoneGroup: a (a) 2026-03-20T11:47:13.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.972+0000 7f48c3739900 4 Zone: a2 (51055576-6662-409d-8abb-ecaeefd3dd80) 2026-03-20T11:47:13.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.972+0000 7f48c3739900 4 using period configuration: cb2fcdce-2c1b-429a-8601-a284ee3a2381:12 2026-03-20T11:47:13.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.972+0000 7f48c3739900 20 period zonegroup name a 2026-03-20T11:47:13.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.972+0000 7f48c3739900 20 using current period zonegroup a 2026-03-20T11:47:13.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.972+0000 7f48c3739900 20 rados->read ofs=0 len=0 2026-03-20T11:47:13.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.972+0000 7f48c3739900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:13.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.972+0000 7f48c3739900 20 rados->read ofs=0 len=0 2026-03-20T11:47:13.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.972+0000 7f48c3739900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:13.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.972+0000 7f48c3739900 20 init(): pipe_map (zone=4b634ebf-58f5-45bc-839a-cb4b425e773a bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:47:13.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.972+0000 7f48c3739900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:13.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.972+0000 7f48c3739900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:13.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.972+0000 7f48c3739900 20 rados->read ofs=0 len=0 2026-03-20T11:47:13.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.973+0000 7f48c3739900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:13.969 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.973+0000 7f48c3739900 20 rados->read ofs=0 len=0 2026-03-20T11:47:13.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.973+0000 7f48c3739900 20 rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:13.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.973+0000 7f48c3739900 20 init(): pipe_map (zone=51055576-6662-409d-8abb-ecaeefd3dd80 bucket=*): adding potential pipe: {id=all,s={b=*,z=},d={b=*,z=}} 2026-03-20T11:47:13.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.973+0000 7f48c3739900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:13.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.973+0000 7f48c3739900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:13.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.973+0000 7f48c3739900 20 reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:13.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.973+0000 7f48c3739900 20 reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:13.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.973+0000 7f48c3739900 20 started sync module instance, tier type = 2026-03-20T11:47:13.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.973+0000 7f48c3739900 20 generating connection object for zone a1 id 4b634ebf-58f5-45bc-839a-cb4b425e773a 2026-03-20T11:47:13.970 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.973+0000 7f48c3739900 20 started zone id=51055576-6662-409d-8abb-ecaeefd3dd80 (name=a2) with tier type = 2026-03-20T11:47:13.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.977+0000 7f48c3739900 20 add_watcher() i=1 2026-03-20T11:47:13.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.977+0000 7f48c3739900 20 add_watcher() i=2 2026-03-20T11:47:13.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.977+0000 7f48c3739900 20 add_watcher() i=3 2026-03-20T11:47:13.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.978+0000 7f48c3739900 20 add_watcher() i=0 2026-03-20T11:47:13.974 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.978+0000 7f48c3739900 20 add_watcher() i=7 2026-03-20T11:47:13.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.978+0000 7f48c3739900 20 add_watcher() i=5 2026-03-20T11:47:13.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.978+0000 7f48c3739900 20 add_watcher() i=6 2026-03-20T11:47:13.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.979+0000 7f48c3739900 20 add_watcher() i=4 2026-03-20T11:47:13.975 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.979+0000 7f48c3739900 2 all 8 watchers are set, enabling cache 2026-03-20T11:47:13.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.981+0000 7f48c3739900 20 rgw_check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2026-03-20T11:47:13.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.981+0000 7f48c3739900 20 rgw_check_secure_mon_conn(): mode 1 is insecure 2026-03-20T11:47:13.978 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:13.981+0000 7f48c3739900 5 note: GC not initialized 2026-03-20T11:47:14.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.039+0000 7f48c3739900 20 init_complete bucket index max shards: 11 2026-03-20T11:47:14.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.039+0000 7f48c3739900 20 Filter name: none 2026-03-20T11:47:14.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.039+0000 7f481ffff640 20 reqs_thread_entry: start 2026-03-20T11:47:14.036 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.039+0000 7f481e7fc640 20 reqs_thread_entry: start 2026-03-20T11:47:14.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.041+0000 7f48c3739900 10 rgw bucket trim manager: Watching bilog.trim 2026-03-20T11:47:14.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.041+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca3130:12BucketTrimCR: operate() 2026-03-20T11:47:14.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.041+0000 7f48c3739900 10 fetching active bucket counters 2026-03-20T11:47:14.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.041+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca5be0:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:14.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.041+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca5be0:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:14.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.041+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is io blocked 2026-03-20T11:47:14.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.041+0000 7f489a7fc640 20 trim: get_bucket_counters: [] 2026-03-20T11:47:14.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.042+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca5be0:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:14.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.042+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca5be0:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:14.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.042+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca5be0:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:14.038 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.042+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca5be0:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:14.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.042+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca3130:12BucketTrimCR: operate() 2026-03-20T11:47:14.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.042+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:14.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.042+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:14.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.042+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is io blocked 2026-03-20T11:47:14.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.042+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:14.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.042+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:14.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.042+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:14.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.042+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:20RGWSimpleRadosReadCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:14.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.042+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca3130:12BucketTrimCR: operate() 2026-03-20T11:47:14.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.042+0000 7f48c3739900 10 listing cold buckets from marker=16:4ca02412:root::dipepu-46:head 2026-03-20T11:47:14.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.042+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:14MetadataListCR: operate() 2026-03-20T11:47:14.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.042+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:14MetadataListCR: operate() 2026-03-20T11:47:14.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.042+0000 7f48c3739900 20 enqueued request req=0x7f4854001250 2026-03-20T11:47:14.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.042+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.042+0000 7f48c3739900 20 req: 0x7f4854001250 2026-03-20T11:47:14.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.042+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is io blocked 2026-03-20T11:47:14.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.042+0000 7f486e7e4640 20 dequeued request req=0x7f4854001250 2026-03-20T11:47:14.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.042+0000 7f486e7e4640 20 RGWWQ: empty 2026-03-20T11:47:14.039 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.042+0000 7f486e7e4640 20 rgw async rados processor: starting metadata listing at 16:4ca02412:root::dipepu-46:head 2026-03-20T11:47:14.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.047+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-46 2026-03-20T11:47:14.044 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.047+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:47:14.048 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.052+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:47:14.052 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.055+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:47:14.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.060+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-26 2026-03-20T11:47:14.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.060+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-20 2026-03-20T11:47:14.057 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.060+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:47:14.060 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.064+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:47:14.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.068+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-1 2026-03-20T11:47:14.064 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.068+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:47:14.068 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.071+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:47:14.072 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.075+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:47:14.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.078+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-52 2026-03-20T11:47:14.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.078+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-32 2026-03-20T11:47:14.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.078+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-42 2026-03-20T11:47:14.075 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.078+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:47:14.078 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.082+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:47:14.081 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.085+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-7 2026-03-20T11:47:14.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.085+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-54 2026-03-20T11:47:14.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.085+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-49 2026-03-20T11:47:14.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.085+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-47 2026-03-20T11:47:14.082 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.085+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:47:14.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.088+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-11 2026-03-20T11:47:14.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.088+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-12 2026-03-20T11:47:14.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.088+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-41 2026-03-20T11:47:14.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.088+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-23 2026-03-20T11:47:14.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.088+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-15 2026-03-20T11:47:14.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.088+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-45 2026-03-20T11:47:14.084 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.088+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:47:14.087 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.090+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:47:14.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.092+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-9 2026-03-20T11:47:14.089 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.092+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:47:14.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.095+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-29 2026-03-20T11:47:14.092 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.095+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:47:14.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.097+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-31 2026-03-20T11:47:14.094 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.097+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:47:14.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.099+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-33 2026-03-20T11:47:14.096 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.099+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:47:14.098 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.101+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-4 2026-03-20T11:47:14.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.101+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:47:14.101 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.103+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:47:14.102 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.105+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:47:14.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.107+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-43 2026-03-20T11:47:14.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.107+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-30 2026-03-20T11:47:14.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.107+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-18 2026-03-20T11:47:14.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.107+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-2 2026-03-20T11:47:14.104 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.107+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:47:14.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.109+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-8 2026-03-20T11:47:14.106 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.109+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:47:14.107 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.111+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:47:14.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.112+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-58 2026-03-20T11:47:14.109 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.112+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:47:14.110 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.113+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:47:14.112 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.115+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-27 2026-03-20T11:47:14.112 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.115+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-28 2026-03-20T11:47:14.112 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.115+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-59 2026-03-20T11:47:14.112 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.115+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:47:14.113 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.116+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:47:14.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.118+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-40 2026-03-20T11:47:14.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.118+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-22 2026-03-20T11:47:14.114 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.118+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:47:14.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.119+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-56 2026-03-20T11:47:14.115 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.119+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:47:14.117 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.120+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:47:14.118 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.121+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:47:14.119 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.122+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-53 2026-03-20T11:47:14.119 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.122+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:47:14.120 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.123+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:47:14.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.124+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:47:14.121 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.125+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:47:14.122 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.125+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:47:14.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.126+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-34 2026-03-20T11:47:14.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.126+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-21 2026-03-20T11:47:14.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.126+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-14 2026-03-20T11:47:14.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.126+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-50 2026-03-20T11:47:14.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.126+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-57 2026-03-20T11:47:14.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.126+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:47:14.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.126+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-39 2026-03-20T11:47:14.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.126+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:47:14.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.127+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-37 2026-03-20T11:47:14.123 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.127+0000 7f486e7e4640 20 rgw async rados processor: restarting metadata listing 2026-03-20T11:47:14.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.134+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-10 2026-03-20T11:47:14.131 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.134+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:47:14.137 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.140+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:47:14.142 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.145+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:47:14.148 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.151+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:47:14.154 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.157+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:47:14.161 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.164+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:47:14.167 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.170+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-19 2026-03-20T11:47:14.167 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.170+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-3 2026-03-20T11:47:14.167 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.170+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:47:14.173 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.176+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:47:14.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.181+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-17 2026-03-20T11:47:14.178 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.181+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:47:14.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.187+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-38 2026-03-20T11:47:14.184 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.187+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:47:14.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.192+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-24 2026-03-20T11:47:14.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.192+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got dipepu-36 2026-03-20T11:47:14.189 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.192+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:47:14.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f486e7e4640 20 rgw async rados processor: RGWRados::pool_iterate: got .bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:47:14.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:14MetadataListCR: operate() 2026-03-20T11:47:14.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:14MetadataListCR: operate() 2026-03-20T11:47:14.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:14MetadataListCR: operate() 2026-03-20T11:47:14.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:14MetadataListCR: operate() 2026-03-20T11:47:14.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca3130:12BucketTrimCR: operate() 2026-03-20T11:47:14.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 4 collected 50 buckets for trim 2026-03-20T11:47:14.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 4 starting trim on bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:47:14.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 4 starting trim on bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:47:14.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4ca9a50:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 4 starting trim on bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:47:14.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4caad20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 4 starting trim on bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:47:14.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca7520:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cabff0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.194 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4caca10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cad430:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca7520:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 enqueued request req=0x55fdc4cadde0 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 req: 0x55fdc4cadde0 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is io blocked 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cabff0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 enqueued request req=0x55fdc4cae180 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 req: 0x55fdc4cadde0 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 req: 0x55fdc4cae180 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is io blocked 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4caca10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 enqueued request req=0x55fdc4cae520 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 req: 0x55fdc4cadde0 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 req: 0x55fdc4cae180 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 req: 0x55fdc4cae520 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is io blocked 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cad430:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 enqueued request req=0x55fdc48e3b90 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 req: 0x55fdc4cadde0 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 req: 0x55fdc4cae180 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 req: 0x55fdc4cae520 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 req: 0x55fdc48e3b90 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is io blocked 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48857fa640 20 dequeued request req=0x55fdc4cadde0 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48857fa640 20 RGWWQ: 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48857fa640 20 req: 0x55fdc4cae180 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48857fa640 20 req: 0x55fdc4cae520 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48857fa640 20 req: 0x55fdc48e3b90 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48857fa640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 : miss 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48857fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f4885ffb640 20 dequeued request req=0x55fdc4cae180 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f4885ffb640 20 RGWWQ: 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f4885ffb640 20 req: 0x55fdc4cae520 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f4885ffb640 20 req: 0x55fdc48e3b90 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f4885ffb640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 : miss 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f4885ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f4884ff9640 20 dequeued request req=0x55fdc4cae520 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f4884ff9640 20 RGWWQ: 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f4884ff9640 20 req: 0x55fdc48e3b90 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f4884ff9640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 : miss 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f4884ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.195 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f487bfff640 20 dequeued request req=0x55fdc48e3b90 2026-03-20T11:47:14.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f487bfff640 20 RGWWQ: empty 2026-03-20T11:47:14.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f487bfff640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 : miss 2026-03-20T11:47:14.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f487bfff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.197+0000 7f48857fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48857fa640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 info.flags=0x17 2026-03-20T11:47:14.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48857fa640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 to cache LRU end 2026-03-20T11:47:14.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48857fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48857fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48857fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:47:14.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48857fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-56 : miss 2026-03-20T11:47:14.196 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48857fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4885ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4885ffb640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 info.flags=0x17 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4885ffb640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 to cache LRU end 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4885ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4885ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4885ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4885ffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-27 : miss 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4885ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4884ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4884ff9640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 info.flags=0x17 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4884ff9640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 to cache LRU end 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4884ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4884ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4884ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4884ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-44 : miss 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4884ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4885ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4885ffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-27 info.flags=0x0 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4885ffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-27 to cache LRU end 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4885ffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-27 : miss 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4885ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48857fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48857fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-56 info.flags=0x0 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48857fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-56 to cache LRU end 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48857fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-56 : miss 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48857fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4885ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4885ffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-27 info.flags=0x0 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4885ffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-27 to cache LRU end 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4885ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32): adding source pipe: {s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4885ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32): adding dest pipe: {s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48857fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48857fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-56 info.flags=0x0 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48857fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-56 to cache LRU end 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48857fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16): adding source pipe: {s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48857fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16): adding dest pipe: {s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4885ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4885ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48857fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4885ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48857fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48857fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cabff0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.197 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cabff0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca7520:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cabff0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca7520:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cabff0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca7520:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca7520:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc48e3fd0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4ca7520:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc48e45f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4ca7520:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4884ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4884ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-44 info.flags=0x0 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4884ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-44 to cache LRU end 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4884ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-44 : miss 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f4884ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f487bfff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f487bfff640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 info.flags=0x17 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f487bfff640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 to cache LRU end 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f487bfff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f487bfff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f487bfff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f487bfff640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-17 : miss 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f487bfff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.198 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 10587b989c4648d000c8bdc92dcb52d8ae2623e06f432c5e1a9cbb8f0e066e72 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:10587b989c4648d000c8bdc92dcb52d8ae2623e06f432c5e1a9cbb8f0e066e72 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 84202489f33844ecab9deebf7e4b34e6f95135077327c446e1e1af9e5a1e9b5e 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=84202489f33844ecab9deebf7e4b34e6f95135077327c446e1e1af9e5a1e9b5e 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-27%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=0, curl_handle=0x55fdc48e6af0 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 run: stack=0x55fdc4ca7c70 is io blocked 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc48e4d80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc48e4d80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.198+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = b5aba1050b57bd1c74849bc313ffad704fcda7f4b604b89beae350660c0ff1a7 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.199 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:b5aba1050b57bd1c74849bc313ffad704fcda7f4b604b89beae350660c0ff1a7 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 2ba78d11e9799eac32a0bbdd06a01c5c3687059c448d6aff51544ce0cb22a8e7 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=2ba78d11e9799eac32a0bbdd06a01c5c3687059c448d6aff51544ce0cb22a8e7 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-56%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=1, curl_handle=0x55fdc4cb95e0 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is io blocked 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f4884ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f481e7fc640 20 link_request req_data=0x55fdc48e75f0 req_data->id=0, curl_handle=0x55fdc48e6af0 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f4884ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-44 info.flags=0x0 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f4884ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-44 to cache LRU end 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f4884ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4): adding source pipe: {s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cb9d30 req_data->id=1, curl_handle=0x55fdc4cb95e0 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f4884ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4): adding dest pipe: {s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f4884ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f4884ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f4884ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4caca10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4caca10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4caca10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4caca10:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4ca9a50:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cbb6c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4ca9a50:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cbbe70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cbbe70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.200 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.201 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.201 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.201 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.201 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.201 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.201 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.201 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.201 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 80a7d9e5e45ffe1d012b052e6173a94cc7d8a2971a458a9db5f86006ef44b48a 2026-03-20T11:47:14.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.201 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.201 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.201 INFO:teuthology.orchestra.run.vm02.stderr:80a7d9e5e45ffe1d012b052e6173a94cc7d8a2971a458a9db5f86006ef44b48a 2026-03-20T11:47:14.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = efaad3269104c819e76133c4ef46f869a1db133fbcc2368d75e48a03dc52bbcd 2026-03-20T11:47:14.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=efaad3269104c819e76133c4ef46f869a1db133fbcc2368d75e48a03dc52bbcd 2026-03-20T11:47:14.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-44%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=2, curl_handle=0x55fdc4cbda00 2026-03-20T11:47:14.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is io blocked 2026-03-20T11:47:14.201 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.199+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cbe150 req_data->id=2, curl_handle=0x55fdc4cbda00 2026-03-20T11:47:14.201 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:47:14.200+0000 7f1249182640 -1 req 2702932635377538079 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:47:14.201 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:47:14.200+0000 7f12bf26e640 -1 req 2893017349486207837 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:47:14.201 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:47:14.200+0000 7f12028f5640 -1 req 3456427740730639370 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:47:14.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f487bfff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f487bfff640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-17 info.flags=0x0 2026-03-20T11:47:14.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f487bfff640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-17 to cache LRU end 2026-03-20T11:47:14.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f487bfff640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-17 : miss 2026-03-20T11:47:14.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f487bfff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f487bfff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f487bfff640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-17 info.flags=0x0 2026-03-20T11:47:14.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f487bfff640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-17 to cache LRU end 2026-03-20T11:47:14.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f487bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22): adding source pipe: {s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f487bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22): adding dest pipe: {s={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f487bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f487bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f487bfff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:47:14.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cad430:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cad430:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cad430:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cad430:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4caad20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 20 cr:s=0x55fdc4cada30:op=0x55fdc4cbf7f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4caad20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 20 cr:s=0x55fdc4cada30:op=0x55fdc4cbfe60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 20 cr:s=0x55fdc4cada30:op=0x55fdc4cbfe60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.205 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 3c03f1bb1a4cfb4f254095aede0b07e328aa06a8bb08b10d3368192158438f4f 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:3c03f1bb1a4cfb4f254095aede0b07e328aa06a8bb08b10d3368192158438f4f 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = c88edb45a7492c9d8cad74889aeccc712f1e05d9ed28d84247ebbf429eef2db5 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c88edb45a7492c9d8cad74889aeccc712f1e05d9ed28d84247ebbf429eef2db5 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-17%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=3, curl_handle=0x55fdc4cc1a40 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f48c3739900 20 run: stack=0x55fdc4cada30 is io blocked 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.200+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cc2150 req_data->id=3, curl_handle=0x55fdc4cc1a40 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4ca7520:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4ca7520:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc48e3fd0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 15 stack 0x55fdc4ca7c70 end 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 run: stack=0x55fdc4ca7c70 is done 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc48e4d80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc48e4d80:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cbbe70:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cbbe70:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.206 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 collect(): s=0x55fdc4bb3e20 stack=0x55fdc4ca7c70 is complete 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 15 stack 0x55fdc4bb3e20 end 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 stack->operate() returned ret=-22 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is done 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc48e45f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 15 stack 0x55fdc48e4c10 end 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is done 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cbb6c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 15 stack 0x55fdc4cbbce0 end 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is done 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 collect(): s=0x55fdc4bc6f60 stack=0x55fdc48e4c10 is complete 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 15 stack 0x55fdc4bc6f60 end 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 stack->operate() returned ret=-22 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is done 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4ca9a50:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 collect(): s=0x55fdc4bbeb40 stack=0x55fdc4cbbce0 is complete 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4ca9a50:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 15 stack 0x55fdc4bbeb40 end 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 stack->operate() returned ret=-22 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is done 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 4 starting trim on bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cb8810:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca9a50:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 4 starting trim on bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.207 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.201+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cb8810:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f48c3739900 20 enqueued request req=0x55fdc4cac740 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f48c3739900 20 req: 0x55fdc4cac740 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is io blocked 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc48e3fb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487b7fe640 20 dequeued request req=0x55fdc4cac740 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487b7fe640 20 RGWWQ: empty 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487b7fe640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 : miss 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f48c3739900 4 starting trim on bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc48e3fb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f48c3739900 20 enqueued request req=0x55fdc48e7020 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f48c3739900 20 req: 0x55fdc48e7020 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is io blocked 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487affd640 20 dequeued request req=0x55fdc48e7020 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487affd640 20 RGWWQ: empty 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487affd640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 : miss 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487affd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc48e4d50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc48e4d50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f48c3739900 20 enqueued request req=0x55fdc4ca84b0 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f48c3739900 20 req: 0x55fdc4ca84b0 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is io blocked 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487a7fc640 20 dequeued request req=0x55fdc4ca84b0 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487a7fc640 20 RGWWQ: empty 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487a7fc640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 : miss 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487a7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487b7fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487b7fe640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 info.flags=0x17 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487b7fe640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 to cache LRU end 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487b7fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487b7fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487b7fe640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487b7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487b7fe640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-46 : miss 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487b7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487b7fe640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-46 info.flags=0x0 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487b7fe640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-46 to cache LRU end 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487b7fe640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-46 : miss 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.208 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487affd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487affd640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 info.flags=0x17 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487affd640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 to cache LRU end 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487affd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487affd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487affd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487affd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-37 : miss 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487affd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487a7fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487a7fc640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 info.flags=0x17 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487a7fc640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 to cache LRU end 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487a7fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487a7fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487a7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487a7fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-1 : miss 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487a7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487b7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487b7fe640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-46 info.flags=0x0 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487b7fe640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-46 to cache LRU end 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6): adding source pipe: {s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.202+0000 7f487b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6): adding dest pipe: {s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487affd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487affd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-37 info.flags=0x0 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487affd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-37 to cache LRU end 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487affd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-37 : miss 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487affd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487a7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487a7fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-1 info.flags=0x0 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487a7fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-1 to cache LRU end 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487a7fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-1 : miss 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487a7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487b7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cb8810:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cb8810:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cb8810:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cb8810:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cbb6a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4ca7520:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4ca7520:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.209 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 90ed47fcb7f2bbe060980ee09d434ec3b2694c2623f391378e8dd0340fd75723 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487affd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487affd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-37 info.flags=0x0 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487affd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-37 to cache LRU end 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487affd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42): adding source pipe: {s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487affd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42): adding dest pipe: {s={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487affd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487affd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:90ed47fcb7f2bbe060980ee09d434ec3b2694c2623f391378e8dd0340fd75723 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487a7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487affd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487a7fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-1 info.flags=0x0 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487a7fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-1 to cache LRU end 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1): adding source pipe: {s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1): adding dest pipe: {s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f487a7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 1e96448fc9d59e6bbcd6541fd2eee37662dba97f57d97b48a23f706b32a99d40 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=1e96448fc9d59e6bbcd6541fd2eee37662dba97f57d97b48a23f706b32a99d40 2026-03-20T11:47:14.210 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-46%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=4, curl_handle=0x55fdc4cbda00 2026-03-20T11:47:14.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is io blocked 2026-03-20T11:47:14.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc48e3fb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc48e4d50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc48e3fb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc48e4d50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc48e3fb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc48e4d50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc48e3fb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc48e4d50:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca9a50:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cbe150 req_data->id=4, curl_handle=0x55fdc4cbda00 2026-03-20T11:47:14.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc48e3fb0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca9a50:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc48e45d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.211 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.211 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:47:14.203+0000 7f12661bc640 -1 req 16640736728844754247 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:47:14.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc48e4d50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc48e5380:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc48e4d50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.212 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.212 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.212 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.212 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.203+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.212 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.212 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.212 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.212 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.212 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.212 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.212 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.212 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.212 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = fedbf77b6378a256568b5b18021b9001a8bb4161ccf10404373e1d694e723e32 2026-03-20T11:47:14.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.212 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.212 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.212 INFO:teuthology.orchestra.run.vm02.stderr:fedbf77b6378a256568b5b18021b9001a8bb4161ccf10404373e1d694e723e32 2026-03-20T11:47:14.212 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = a5deaa124c467a99d72d528976232c8a861f92f34e604e27ce24985554becf83 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a5deaa124c467a99d72d528976232c8a861f92f34e604e27ce24985554becf83 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-37%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=5, curl_handle=0x55fdc4cb95e0 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is io blocked 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc48e5380:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cb9d30 req_data->id=5, curl_handle=0x55fdc4cb95e0 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 5d4959a535adb22050503e948ed62d0af3dc92ecc8abe4809228c02da6ae2214 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:5d4959a535adb22050503e948ed62d0af3dc92ecc8abe4809228c02da6ae2214 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 029fdb5614df3e8f08399ac7e0639892dd95a79341c6284cff6cb87d35fa5e97 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=029fdb5614df3e8f08399ac7e0639892dd95a79341c6284cff6cb87d35fa5e97 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.213 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-1%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=6, curl_handle=0x55fdc48e6af0 2026-03-20T11:47:14.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f48c3739900 20 run: stack=0x55fdc4ca7c70 is io blocked 2026-03-20T11:47:14.214 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.204+0000 7f481e7fc640 20 link_request req_data=0x55fdc48e75f0 req_data->id=6, curl_handle=0x55fdc48e6af0 2026-03-20T11:47:14.214 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:47:14.205+0000 7f12b3a57640 -1 req 4690173384683632076 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:47:14.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc48e5380:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc48e5380:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:14.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc48e45d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f48c3739900 15 stack 0x55fdc4ca7c70 end 2026-03-20T11:47:14.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f48c3739900 20 run: stack=0x55fdc4ca7c70 is done 2026-03-20T11:47:14.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f48c3739900 20 collect(): s=0x55fdc4bbeb40 stack=0x55fdc4ca7c70 is complete 2026-03-20T11:47:14.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f48c3739900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:14.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f48c3739900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:14.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:14.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f48c3739900 15 stack 0x55fdc4bbeb40 end 2026-03-20T11:47:14.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f48c3739900 20 stack->operate() returned ret=-22 2026-03-20T11:47:14.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is done 2026-03-20T11:47:14.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f48c3739900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:14.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f48c3739900 4 starting trim on bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:47:14.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cc39b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cc39b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f48c3739900 20 enqueued request req=0x55fdc48e7020 2026-03-20T11:47:14.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f48c3739900 20 req: 0x55fdc48e7020 2026-03-20T11:47:14.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is io blocked 2026-03-20T11:47:14.215 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f4879ffb640 20 dequeued request req=0x55fdc48e7020 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f4879ffb640 20 RGWWQ: empty 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f4879ffb640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 : miss 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.205+0000 7f4879ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f4879ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f4879ffb640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 info.flags=0x17 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f4879ffb640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 to cache LRU end 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f4879ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f4879ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f4879ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f4879ffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-7 : miss 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f4879ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f4879ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f4879ffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-7 info.flags=0x0 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f4879ffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-7 to cache LRU end 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f4879ffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-7 : miss 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f4879ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f4879ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f4879ffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-7 info.flags=0x0 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f4879ffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-7 to cache LRU end 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f4879ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12): adding source pipe: {s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f4879ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12): adding dest pipe: {s={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f4879ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f4879ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f4879ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cc39b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cc39b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cc39b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cc39b0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc48e5380:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cc39b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cc39b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.206+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.207+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.207+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.207+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.207+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.207+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.216 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.217 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.217 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.217 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.217 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.217 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.217 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.217 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.207+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = d1f591c35cd3077b42b1790a00cff2aa6c6f7ac8b880bf67d70272a249049a58 2026-03-20T11:47:14.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.207+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.217 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.217 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.217 INFO:teuthology.orchestra.run.vm02.stderr:d1f591c35cd3077b42b1790a00cff2aa6c6f7ac8b880bf67d70272a249049a58 2026-03-20T11:47:14.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.207+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.207+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.207+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.207+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.207+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 302ec61c8191c5c829a90c7546c13268e33763a798b55fbe16bedbbac80987cf 2026-03-20T11:47:14.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.207+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=302ec61c8191c5c829a90c7546c13268e33763a798b55fbe16bedbbac80987cf 2026-03-20T11:47:14.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.207+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.207+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.207+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-7%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.207+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=7, curl_handle=0x55fdc48e6af0 2026-03-20T11:47:14.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.207+0000 7f48c3739900 20 run: stack=0x55fdc4ca7c70 is io blocked 2026-03-20T11:47:14.217 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.207+0000 7f481e7fc640 20 link_request req_data=0x55fdc48e75f0 req_data->id=7, curl_handle=0x55fdc48e6af0 2026-03-20T11:47:14.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.242+0000 7f48c3739900 20 cr:s=0x55fdc4cada30:op=0x55fdc4cbfe60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.239 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.242+0000 7f48c3739900 20 cr:s=0x55fdc4cada30:op=0x55fdc4cbfe60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.242+0000 7f48c3739900 20 cr:s=0x55fdc4cada30:op=0x55fdc4cbfe60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.242+0000 7f48c3739900 20 cr:s=0x55fdc4cada30:op=0x55fdc4cbfe60:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.242+0000 7f48c3739900 20 cr:s=0x55fdc4cada30:op=0x55fdc4cbf7f0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.242+0000 7f48c3739900 15 stack 0x55fdc4cada30 end 2026-03-20T11:47:14.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.242+0000 7f48c3739900 20 run: stack=0x55fdc4cada30 is done 2026-03-20T11:47:14.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.242+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4caad20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.242+0000 7f48c3739900 20 collect(): s=0x55fdc4ca7e70 stack=0x55fdc4cada30 is complete 2026-03-20T11:47:14.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.242+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.242+0000 7f48c3739900 10 trimming bilogs for bucket=:dipepu-17[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22]) markers=[,,,,,,,,00000000005.51.18,,], shards=11 2026-03-20T11:47:14.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.242+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cf42e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.242+0000 7f48c3739900 10 trimming bilog shard 8 of :dipepu-17[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22]) at marker 00000000005.51.18 2026-03-20T11:47:14.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.242+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.242+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4d02820:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.242+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4d02820:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.242+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22.8 2026-03-20T11:47:14.240 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.242+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is io blocked 2026-03-20T11:47:14.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4ca7520:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4ca7520:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:14.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cbb6a0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f48c3739900 15 stack 0x55fdc4cbbce0 end 2026-03-20T11:47:14.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is done 2026-03-20T11:47:14.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f48c3739900 20 collect(): s=0x55fdc4bb3e20 stack=0x55fdc4cbbce0 is complete 2026-03-20T11:47:14.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f48c3739900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:14.241 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f48c3739900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:14.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:14.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f48c3739900 15 stack 0x55fdc4bb3e20 end 2026-03-20T11:47:14.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f48c3739900 20 stack->operate() returned ret=-22 2026-03-20T11:47:14.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is done 2026-03-20T11:47:14.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f48c3739900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:14.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f48c3739900 4 starting trim on bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:47:14.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cc6600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cc6600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f48c3739900 20 enqueued request req=0x55fdc4cac740 2026-03-20T11:47:14.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f48c3739900 20 req: 0x55fdc4cac740 2026-03-20T11:47:14.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is io blocked 2026-03-20T11:47:14.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f4878ff9640 20 dequeued request req=0x55fdc4cac740 2026-03-20T11:47:14.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f4878ff9640 20 RGWWQ: empty 2026-03-20T11:47:14.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f4878ff9640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 : miss 2026-03-20T11:47:14.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f4878ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.242 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4d02820:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f4878ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f4878ff9640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 info.flags=0x17 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f4878ff9640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 to cache LRU end 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f4878ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f4878ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f4878ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.244+0000 7f4878ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f4878ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-48 : miss 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f4878ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4d02820:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f48c3739900 15 stack 0x55fdc4cf32a0 end 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is done 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cf42e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4caad20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f48c3739900 20 trim: trimmed bucket instance dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f48c3739900 15 stack 0x55fdc4ca7e70 end 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is done 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f4878ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f4878ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-48 info.flags=0x0 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f4878ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-48 to cache LRU end 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f4878ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-48 : miss 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f4878ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4caad20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f48c3739900 4 starting trim on bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cf2ac0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cf2ac0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f48c3739900 20 enqueued request req=0x55fdc48e3b90 2026-03-20T11:47:14.245 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f48c3739900 20 req: 0x55fdc48e3b90 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is io blocked 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f48787f8640 20 dequeued request req=0x55fdc48e3b90 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f48787f8640 20 RGWWQ: empty 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f48787f8640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 : miss 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f48787f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f4878ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f4878ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-48 info.flags=0x0 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f4878ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-48 to cache LRU end 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f4878ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8): adding source pipe: {s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f4878ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8): adding dest pipe: {s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f4878ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f4878ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.245+0000 7f4878ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48787f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48787f8640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 info.flags=0x17 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48787f8640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 to cache LRU end 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48787f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48787f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48787f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48787f8640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-59 : miss 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48787f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cc6600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cc6600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc48e4d50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48787f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48787f8640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-59 info.flags=0x0 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48787f8640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-59 to cache LRU end 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48787f8640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-59 : miss 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48787f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cc6600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc48e4d50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cc6600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc48e4d50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48787f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48787f8640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-59 info.flags=0x0 2026-03-20T11:47:14.246 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48787f8640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-59 to cache LRU end 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48787f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19): adding source pipe: {s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48787f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19): adding dest pipe: {s={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48787f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48787f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48787f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc48e4d50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cfc150:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cf2ac0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc48e3fb0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 15 stack 0x55fdc48e4c10 end 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is done 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc48e4d50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cf2ac0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca9a50:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 20 collect(): s=0x55fdc4bc6f60 stack=0x55fdc48e4c10 is complete 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 10 trimming bilogs for bucket=:dipepu-37[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42]) markers=[,,,,,,,,,,00000000001.45.6], shards=11 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc48e4d50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.247 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 63033d278de0a4f3b6ea17d7f2555aed980c960262f78dea8e81502df3751608 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.246+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:63033d278de0a4f3b6ea17d7f2555aed980c960262f78dea8e81502df3751608 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 5851233b675f8b45ce8ae6fdeaaf7d5cfd00aa510e9d57348141c99594ea393d 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=5851233b675f8b45ce8ae6fdeaaf7d5cfd00aa510e9d57348141c99594ea393d 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-48%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=8, curl_handle=0x55fdc4cb95e0 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is io blocked 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cf2ac0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc48e3f10:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 10 trimming bilog shard 10 of :dipepu-37[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42]) at marker 00000000001.45.6 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cf2ac0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4cf3440:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4caad20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cb9d30 req_data->id=8, curl_handle=0x55fdc4cb95e0 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4cf3440:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42.10 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is io blocked 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cfb920:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4caad20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cc6600:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cc6600:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.248 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 2ab9161e49f65a2d6a6486c2cbbdcd7849861ae15689a84c20721ca6a9881321 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr:2ab9161e49f65a2d6a6486c2cbbdcd7849861ae15689a84c20721ca6a9881321 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 594fc68eccecc57789c50af1442ee0b3b5a541ea8bff5f264eefea168b656bff 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=594fc68eccecc57789c50af1442ee0b3b5a541ea8bff5f264eefea168b656bff 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-59%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=9, curl_handle=0x55fdc4cbda00 2026-03-20T11:47:14.249 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is io blocked 2026-03-20T11:47:14.249 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:47:14.247+0000 7f12729d5640 -1 req 8022385183011685669 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:47:14.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.247+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cbe150 req_data->id=9, curl_handle=0x55fdc4cbda00 2026-03-20T11:47:14.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.249+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4cf3440:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.249+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4cf3440:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.249+0000 7f48c3739900 15 stack 0x55fdc48e4c10 end 2026-03-20T11:47:14.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.249+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.249+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is done 2026-03-20T11:47:14.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.249+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc48e3f10:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.249+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca9a50:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.249+0000 7f48c3739900 20 trim: trimmed bucket instance dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42 2026-03-20T11:47:14.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.249+0000 7f48c3739900 15 stack 0x55fdc4bc6f60 end 2026-03-20T11:47:14.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.249+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is done 2026-03-20T11:47:14.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.249+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.249+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.249+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca9a50:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.249+0000 7f48c3739900 4 starting trim on bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:47:14.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.249+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cf2ac0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.249+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cf2ac0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.249+0000 7f48c3739900 20 enqueued request req=0x55fdc4cac740 2026-03-20T11:47:14.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.249+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.249+0000 7f48c3739900 20 req: 0x55fdc4cac740 2026-03-20T11:47:14.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.249+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is io blocked 2026-03-20T11:47:14.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.249+0000 7f4877ff7640 20 dequeued request req=0x55fdc4cac740 2026-03-20T11:47:14.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.249+0000 7f4877ff7640 20 RGWWQ: empty 2026-03-20T11:47:14.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.249+0000 7f4877ff7640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 : miss 2026-03-20T11:47:14.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.249+0000 7f4877ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.254 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cc39b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f4877ff7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f4877ff7640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 info.flags=0x17 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f4877ff7640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 to cache LRU end 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f4877ff7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f4877ff7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f4877ff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f4877ff7640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-21 : miss 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f4877ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cc39b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cc39b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cc39b0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc48e5380:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 15 stack 0x55fdc4ca7c70 end 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 20 run: stack=0x55fdc4ca7c70 is done 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 20 collect(): s=0x55fdc4bbeb40 stack=0x55fdc4ca7c70 is complete 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 10 trimming bilogs for bucket=:dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) markers=[,00000000002.6.6,,00000000001.11.6,,00000000001.7.6,00000000002.9.6,00000000001.8.6,00000000001.12.6,00000000001.7.6,], shards=11 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc48e5380:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 10 trimming bilog shard 1 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000002.6.6 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 10 trimming bilog shard 3 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.11.6 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 10 trimming bilog shard 5 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.7.6 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 10 trimming bilog shard 6 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000002.9.6 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 10 trimming bilog shard 7 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.8.6 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f4877ff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 10 trimming bilog shard 8 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.12.6 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f4877ff7640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-21 info.flags=0x0 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f4877ff7640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-21 to cache LRU end 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f4877ff7640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-21 : miss 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f4877ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 10 trimming bilog shard 9 of :dipepu-7[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12]) at marker 00000000001.7.6 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cf8090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4cf9450:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cc80a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4cf6bf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 20 cr:s=0x55fdc4cf77d0:op=0x55fdc4cfcfa0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cc4870:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4d00940:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cf8090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.1 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 20 run: stack=0x55fdc4ca7c70 is io blocked 2026-03-20T11:47:14.255 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4cf9450:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.3 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is io blocked 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f4877ff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f4877ff7640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-21 info.flags=0x0 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f4877ff7640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-21 to cache LRU end 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f4877ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26): adding source pipe: {s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f4877ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26): adding dest pipe: {s={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cc80a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f4877ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f4877ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.5 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.250+0000 7f4877ff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.251+0000 7f48c3739900 20 run: stack=0x55fdc4d01f90 is io blocked 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.251+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4cf6bf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.251+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.6 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.251+0000 7f48c3739900 20 run: stack=0x55fdc4d025f0 is io blocked 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.251+0000 7f48c3739900 20 cr:s=0x55fdc4cf77d0:op=0x55fdc4cfcfa0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.251+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.7 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.251+0000 7f48c3739900 20 run: stack=0x55fdc4cf77d0 is io blocked 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.252+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cc4870:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.252+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.8 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.252+0000 7f48c3739900 20 run: stack=0x55fdc4ca7520 is io blocked 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.252+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4d00940:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.252+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12.9 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.252+0000 7f48c3739900 20 run: stack=0x55fdc4cf64d0 is io blocked 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.252+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cf2ac0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.252+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cf8090:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.252+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cf8090:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.252+0000 7f48c3739900 15 stack 0x55fdc4ca7c70 end 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.252+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.252+0000 7f48c3739900 20 run: stack=0x55fdc4ca7c70 is done 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.252+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cf2ac0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.252+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc48e5380:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.252+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.252+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cf2ac0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.252+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cf2ac0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.252+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca9a50:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.252+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cffd50:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.252+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca9a50:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.252+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.252+0000 7f48c3739900 20 cr:s=0x55fdc4cf77d0:op=0x55fdc4cfcfa0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.256 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.252+0000 7f48c3739900 20 cr:s=0x55fdc4cf77d0:op=0x55fdc4cfcfa0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.252+0000 7f48c3739900 15 stack 0x55fdc4cf77d0 end 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 20 run: stack=0x55fdc4cf77d0 is done 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cf7980:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4cf6bf0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4cf6bf0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 15 stack 0x55fdc4d025f0 end 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 20 run: stack=0x55fdc4d025f0 is done 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4cf9450:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4cf9450:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 15 stack 0x55fdc48e4c10 end 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is done 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc48e5380:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cf7980:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 46642177d589670c8ae3c22630109107dcb63b7dcbce409d790aa08e37fc5c8b 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:46642177d589670c8ae3c22630109107dcb63b7dcbce409d790aa08e37fc5c8b 2026-03-20T11:47:14.257 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 47563f12057344ea31d8f09ec671d4170c70f4cf3547dcfe4a4f9658c40dcd07 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=47563f12057344ea31d8f09ec671d4170c70f4cf3547dcfe4a4f9658c40dcd07 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-21%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=10, curl_handle=0x55fdc48e6af0 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 20 run: stack=0x55fdc4ca7c70 is io blocked 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc48e5380:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f481e7fc640 20 link_request req_data=0x55fdc48e75f0 req_data->id=10, curl_handle=0x55fdc48e6af0 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc48e5380:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.253+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.254+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4d00940:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.254+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4d00940:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.254+0000 7f48c3739900 15 stack 0x55fdc4cf64d0 end 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.254+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.254+0000 7f48c3739900 20 run: stack=0x55fdc4cf64d0 is done 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.254+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc48e5380:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.254+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.254+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cc80a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.254+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cc80a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.254+0000 7f48c3739900 15 stack 0x55fdc4d01f90 end 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.254+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.254+0000 7f48c3739900 20 run: stack=0x55fdc4d01f90 is done 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.254+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc48e5380:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.254+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.254+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cc4870:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.254+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cc4870:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.254+0000 7f48c3739900 15 stack 0x55fdc4ca7520 end 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.254+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.254+0000 7f48c3739900 20 run: stack=0x55fdc4ca7520 is done 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.254+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc48e5380:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.254+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.254+0000 7f48c3739900 20 trim: trimmed bucket instance dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.254+0000 7f48c3739900 15 stack 0x55fdc4bbeb40 end 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is done 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.258 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48c3739900 4 starting trim on bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4d017c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4d017c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48c3739900 20 enqueued request req=0x55fdc4cc0410 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48c3739900 20 req: 0x55fdc4cc0410 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is io blocked 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48777f6640 20 dequeued request req=0x55fdc4cc0410 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48777f6640 20 RGWWQ: empty 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48777f6640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 : miss 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48777f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48777f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48777f6640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 info.flags=0x17 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48777f6640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 to cache LRU end 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48777f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48777f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48777f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48777f6640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-41 : miss 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48777f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48777f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48777f6640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-41 info.flags=0x0 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48777f6640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-41 to cache LRU end 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48777f6640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-41 : miss 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48777f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48777f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48777f6640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-41 info.flags=0x0 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48777f6640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-41 to cache LRU end 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48777f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1): adding source pipe: {s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48777f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1): adding dest pipe: {s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48777f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48777f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48777f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4d017c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4d017c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4d017c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4d017c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cc38d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.255+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.256+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.256+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cbf7f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.256+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cbf7f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.256+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.259 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.256+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.256+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.256+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.256+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.256+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.256+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 2d989b6b922d2f62e9b72ac8caf689e08f6cdc19120d32669accb40b3133e13a 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.256+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:2d989b6b922d2f62e9b72ac8caf689e08f6cdc19120d32669accb40b3133e13a 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.256+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.256+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.256+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.256+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.256+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = d63be4436a3d72f1584d11873d63592aa1ddf9158455d981095e7809882024fd 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.256+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=d63be4436a3d72f1584d11873d63592aa1ddf9158455d981095e7809882024fd 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.256+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.256+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.256+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-41%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.256+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=11, curl_handle=0x55fdc4cc1a40 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.256+0000 7f48c3739900 20 run: stack=0x55fdc4ca7520 is io blocked 2026-03-20T11:47:14.260 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.256+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cc2150 req_data->id=11, curl_handle=0x55fdc4cc1a40 2026-03-20T11:47:14.261 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:47:14.256+0000 7f12bc268640 -1 req 5659187005502029409 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:47:14.286 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc48e4d50:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.286 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc48e4d50:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:14.286 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cfc150:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.286 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f48c3739900 15 stack 0x55fdc4cf32a0 end 2026-03-20T11:47:14.286 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is done 2026-03-20T11:47:14.286 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.286 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f48c3739900 20 collect(): s=0x55fdc4bb3e20 stack=0x55fdc4cf32a0 is complete 2026-03-20T11:47:14.286 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.286 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f48c3739900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:14.286 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f48c3739900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:14.286 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:14.286 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f48c3739900 15 stack 0x55fdc4bb3e20 end 2026-03-20T11:47:14.286 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f48c3739900 20 stack->operate() returned ret=-22 2026-03-20T11:47:14.286 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is done 2026-03-20T11:47:14.286 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.286 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f48c3739900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:14.286 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.286 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.286 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f48c3739900 4 starting trim on bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:47:14.286 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4d017c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.286 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4d017c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.286 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f48c3739900 20 enqueued request req=0x55fdc4cae520 2026-03-20T11:47:14.286 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.286 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f48c3739900 20 req: 0x55fdc4cae520 2026-03-20T11:47:14.286 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is io blocked 2026-03-20T11:47:14.287 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f4876ff5640 20 dequeued request req=0x55fdc4cae520 2026-03-20T11:47:14.287 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f4876ff5640 20 RGWWQ: empty 2026-03-20T11:47:14.287 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f4876ff5640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 : miss 2026-03-20T11:47:14.287 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.288+0000 7f4876ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.287 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.289+0000 7f4876ff5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:47:14.287 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.289+0000 7f4876ff5640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 info.flags=0x17 2026-03-20T11:47:14.287 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.289+0000 7f4876ff5640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 to cache LRU end 2026-03-20T11:47:14.287 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.289+0000 7f4876ff5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.287 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.289+0000 7f4876ff5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.287 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.289+0000 7f4876ff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:47:14.287 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.289+0000 7f4876ff5640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-49 : miss 2026-03-20T11:47:14.287 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.289+0000 7f4876ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.289+0000 7f4876ff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.289+0000 7f4876ff5640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-49 info.flags=0x0 2026-03-20T11:47:14.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.289+0000 7f4876ff5640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-49 to cache LRU end 2026-03-20T11:47:14.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.289+0000 7f4876ff5640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-49 : miss 2026-03-20T11:47:14.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.289+0000 7f4876ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cc6600:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f4876ff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f4876ff5640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-49 info.flags=0x0 2026-03-20T11:47:14.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f4876ff5640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-49 to cache LRU end 2026-03-20T11:47:14.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f4876ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9): adding source pipe: {s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f4876ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9): adding dest pipe: {s={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f4876ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.288 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f4876ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f4876ff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cc6600:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4d017c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cc6600:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4d017c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cc6600:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4d017c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cfb920:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f48c3739900 15 stack 0x55fdc4cbbce0 end 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is done 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4d017c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4caad20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f48c3739900 20 collect(): s=0x55fdc4ca7e70 stack=0x55fdc4cbbce0 is complete 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f48c3739900 10 trimming bilogs for bucket=:dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) markers=[,00000000001.65.6,,,,00000000001.44.6,,,00000000001.58.6,,00000000001.63.6], shards=11 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cf59e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f48c3739900 10 trimming bilog shard 1 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.65.6 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f48c3739900 10 trimming bilog shard 5 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.44.6 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f48c3739900 10 trimming bilog shard 8 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.58.6 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f48c3739900 10 trimming bilog shard 10 of :dipepu-59[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19]) at marker 00000000001.63.6 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.290+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.291+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cf9c80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.291+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.291+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.291+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cf8070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.291+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cbbe20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.291+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4d02900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.291+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4cc5930:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.291+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cfb920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.291+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cf8070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.291+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.1 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.291+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is io blocked 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.291+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cbbe20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.291+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.5 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.291+0000 7f48c3739900 20 run: stack=0x55fdc4d01f90 is io blocked 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.291+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4d02900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.291+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.8 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.291+0000 7f48c3739900 20 run: stack=0x55fdc4cf64d0 is io blocked 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.291+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4cc5930:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.289 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.291+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19.10 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.292+0000 7f48c3739900 20 run: stack=0x55fdc4d025f0 is io blocked 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.292+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cfb920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.292+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.292+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.292+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.292+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.292+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.292+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.292+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = d27289ed70771db2e2e1c6e57d1cb496d2c5eb091e78a0244a53cdd2708ced17 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.292+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:d27289ed70771db2e2e1c6e57d1cb496d2c5eb091e78a0244a53cdd2708ced17 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.292+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.292+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.292+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.292+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.292+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 0e374774d939565e786dab746ec37e07f9117f67423cd1c000ca9bde32a0dc0f 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.292+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=0e374774d939565e786dab746ec37e07f9117f67423cd1c000ca9bde32a0dc0f 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.292+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.293+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.293+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-49%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.293+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=12, curl_handle=0x55fdc4cbda00 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.293+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is io blocked 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.293+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cbbe20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.293+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cbe150 req_data->id=12, curl_handle=0x55fdc4cbda00 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.293+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cbbe20:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.293+0000 7f48c3739900 15 stack 0x55fdc4d01f90 end 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.293+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.292 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.293+0000 7f48c3739900 20 run: stack=0x55fdc4d01f90 is done 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.293+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cf8070:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.293+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cf8070:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.293+0000 7f48c3739900 15 stack 0x55fdc4cf32a0 end 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.293+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.293+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is done 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.293+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cf59e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.293+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4cc5930:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.293+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4cc5930:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.293+0000 7f48c3739900 15 stack 0x55fdc4d025f0 end 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.293+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.293+0000 7f48c3739900 20 run: stack=0x55fdc4d025f0 is done 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.293+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4d02900:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.294+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4d02900:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.294+0000 7f48c3739900 15 stack 0x55fdc4cf64d0 end 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.294+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.294+0000 7f48c3739900 20 run: stack=0x55fdc4cf64d0 is done 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.294+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cf59e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.294+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cf59e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.294+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cf59e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.294+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4caad20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.294+0000 7f48c3739900 20 trim: trimmed bucket instance dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.294+0000 7f48c3739900 15 stack 0x55fdc4ca7e70 end 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.294+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is done 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.294+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.294+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.294+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4caad20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.294+0000 7f48c3739900 4 starting trim on bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.294+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d017c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.294+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d017c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.294+0000 7f48c3739900 20 enqueued request req=0x55fdc4cc6130 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.294+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.294+0000 7f48c3739900 20 req: 0x55fdc4cc6130 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.294+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is io blocked 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.294+0000 7f48767f4640 20 dequeued request req=0x55fdc4cc6130 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.294+0000 7f48767f4640 20 RGWWQ: empty 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.294+0000 7f48767f4640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 : miss 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.294+0000 7f48767f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.295+0000 7f48767f4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.295+0000 7f48767f4640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 info.flags=0x17 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.295+0000 7f48767f4640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 to cache LRU end 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.295+0000 7f48767f4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.295+0000 7f48767f4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.295+0000 7f48767f4640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:47:14.293 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.295+0000 7f48767f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:47:14.294 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.295+0000 7f48767f4640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-45 : miss 2026-03-20T11:47:14.294 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.295+0000 7f48767f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.295+0000 7f48767f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.295+0000 7f48767f4640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-45 info.flags=0x0 2026-03-20T11:47:14.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.295+0000 7f48767f4640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-45 to cache LRU end 2026-03-20T11:47:14.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.295+0000 7f48767f4640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-45 : miss 2026-03-20T11:47:14.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.295+0000 7f48767f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.295+0000 7f48767f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.295+0000 7f48767f4640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-45 info.flags=0x0 2026-03-20T11:47:14.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.295+0000 7f48767f4640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-45 to cache LRU end 2026-03-20T11:47:14.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.295+0000 7f48767f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5): adding source pipe: {s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.295+0000 7f48767f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5): adding dest pipe: {s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.295+0000 7f48767f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.295+0000 7f48767f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.295+0000 7f48767f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5 2026-03-20T11:47:14.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.295+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d017c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.295+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d017c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.295+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d017c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d017c0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4caad20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.295 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cf7980:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4cc6600:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4caad20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cf7980:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cf7980:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 3f80614127a9297bb02b74a823e77507a35a617fe9179970579418a9688f1a91 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:3f80614127a9297bb02b74a823e77507a35a617fe9179970579418a9688f1a91 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 251005bda50f65859cd2a99913cb371867b1ca47896f4ad9a8eab972672fbdff 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=251005bda50f65859cd2a99913cb371867b1ca47896f4ad9a8eab972672fbdff 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-45%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=13, curl_handle=0x55fdc48e6af0 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 20 run: stack=0x55fdc4d025f0 is io blocked 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f481e7fc640 20 link_request req_data=0x55fdc48e75f0 req_data->id=13, curl_handle=0x55fdc48e6af0 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cf7980:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.296+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cffd50:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.297+0000 7f48c3739900 15 stack 0x55fdc4ca7c70 end 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.297+0000 7f48c3739900 20 run: stack=0x55fdc4ca7c70 is done 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.297+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca9a50:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.296 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.297+0000 7f48c3739900 20 collect(): s=0x55fdc4bc6f60 stack=0x55fdc4ca7c70 is complete 2026-03-20T11:47:14.297 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:47:14.297+0000 7f121691d640 -1 req 346833682045866605 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.297+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.297+0000 7f48c3739900 10 trimming bilogs for bucket=:dipepu-21[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26]) markers=[,,,,,,,,00000000005.84.14,,], shards=11 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.297+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cbf7f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.297+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cbf7f0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.297+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cf7980:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.297+0000 7f48c3739900 10 trimming bilog shard 8 of :dipepu-21[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26]) at marker 00000000005.84.14 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.297+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.297+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cc38d0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.297+0000 7f48c3739900 15 stack 0x55fdc4ca7520 end 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.297+0000 7f48c3739900 20 run: stack=0x55fdc4ca7520 is done 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.297+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cc5930:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.297+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.297+0000 7f48c3739900 20 collect(): s=0x55fdc4bbeb40 stack=0x55fdc4ca7520 is complete 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.297+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.297+0000 7f48c3739900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.297+0000 7f48c3739900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.297+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.297+0000 7f48c3739900 15 stack 0x55fdc4bbeb40 end 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.297+0000 7f48c3739900 20 stack->operate() returned ret=-22 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.297+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is done 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.297+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cc5930:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.298+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26.8 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.298+0000 7f48c3739900 20 run: stack=0x55fdc4ca7c70 is io blocked 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.298+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.298+0000 7f48c3739900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.298+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.298+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4d09bc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.298+0000 7f48c3739900 4 starting trim on bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.298+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cf4090:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.298+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cf4090:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.298+0000 7f48c3739900 20 enqueued request req=0x55fdc4cac740 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f48c3739900 20 req: 0x55fdc4cac740 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is io blocked 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f4875ff3640 20 dequeued request req=0x55fdc4cac740 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f4875ff3640 20 RGWWQ: empty 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f4875ff3640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 : miss 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f4875ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cc5930:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.299 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f4875ff3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f4875ff3640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 info.flags=0x17 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f4875ff3640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 to cache LRU end 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f4875ff3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f4875ff3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f4875ff3640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f4875ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f4875ff3640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-33 : miss 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f4875ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cc5930:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f48c3739900 15 stack 0x55fdc4ca7c70 end 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f48c3739900 20 run: stack=0x55fdc4ca7c70 is done 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cf7980:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca9a50:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f48c3739900 20 trim: trimmed bucket instance dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f4875ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f4875ff3640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-33 info.flags=0x0 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f4875ff3640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-33 to cache LRU end 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f4875ff3640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-33 : miss 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f4875ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f48c3739900 15 stack 0x55fdc4bc6f60 end 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.299+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is done 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 4 starting trim on bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cf5140:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cf5140:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 20 enqueued request req=0x55fdc4cc0410 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 20 req: 0x55fdc4cc0410 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is io blocked 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48757f2640 20 dequeued request req=0x55fdc4cc0410 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48757f2640 20 RGWWQ: empty 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48757f2640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 : miss 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48757f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f4875ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f4875ff3640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-33 info.flags=0x0 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f4875ff3640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-33 to cache LRU end 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f4875ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38): adding source pipe: {s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f4875ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38): adding dest pipe: {s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f4875ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.300 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f4875ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.300 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:47:14.300+0000 7f11fb8e7640 -1 req 4350886644152693055 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f4875ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cf4090:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cf4090:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cf4090:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cf4090:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4d09bc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cf7980:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4d09bc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cffd10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cffd10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.301 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 5aa4c6c43af820ec24ee78d4ab0fe9552adda62509e9155aac49eb277ea85d82 2026-03-20T11:47:14.302 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.302 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.302 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.302 INFO:teuthology.orchestra.run.vm02.stderr:5aa4c6c43af820ec24ee78d4ab0fe9552adda62509e9155aac49eb277ea85d82 2026-03-20T11:47:14.302 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.302 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.302 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.302 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.302 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = ea47074c2be34cb77319b85e9c9ad4ba94fb23493cd0ff32e48b2258f65a0e22 2026-03-20T11:47:14.302 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=ea47074c2be34cb77319b85e9c9ad4ba94fb23493cd0ff32e48b2258f65a0e22 2026-03-20T11:47:14.302 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.302 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.302 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-33%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.302 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=14, curl_handle=0x55fdc4cc1a40 2026-03-20T11:47:14.302 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:47:14.301+0000 7f12cca89640 -1 req 11906374111372027749 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:47:14.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48c3739900 20 run: stack=0x55fdc4ca7c70 is io blocked 2026-03-20T11:47:14.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cc2150 req_data->id=14, curl_handle=0x55fdc4cc1a40 2026-03-20T11:47:14.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48757f2640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:47:14.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48757f2640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 info.flags=0x17 2026-03-20T11:47:14.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48757f2640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 to cache LRU end 2026-03-20T11:47:14.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48757f2640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48757f2640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48757f2640 10 rgw async rados processor: updating xattr: name=user.rgw.iam-policy bl.length()=97 2026-03-20T11:47:14.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48757f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:47:14.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48757f2640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-47 : miss 2026-03-20T11:47:14.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48757f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48757f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48757f2640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-47 info.flags=0x0 2026-03-20T11:47:14.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48757f2640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-47 to cache LRU end 2026-03-20T11:47:14.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48757f2640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-47 : miss 2026-03-20T11:47:14.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.300+0000 7f48757f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48757f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48757f2640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-47 info.flags=0x0 2026-03-20T11:47:14.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48757f2640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-47 to cache LRU end 2026-03-20T11:47:14.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48757f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7): adding source pipe: {s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.303 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48757f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7): adding dest pipe: {s={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48757f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48757f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48757f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cf5140:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cf5140:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cf5140:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cf5140:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4d05090:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cc7f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cc7f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = e0cfa5fd2a1223eeb868754c1458471bcf5f17ad32169fa14fb7393027827ed3 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:e0cfa5fd2a1223eeb868754c1458471bcf5f17ad32169fa14fb7393027827ed3 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = dd523c6af9bce5fa17a608fd4ec8df80f229bddc93f84708277472e79a275b97 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=dd523c6af9bce5fa17a608fd4ec8df80f229bddc93f84708277472e79a275b97 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-47%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=15, curl_handle=0x55fdc4cb95e0 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f48c3739900 20 run: stack=0x55fdc4ca7520 is io blocked 2026-03-20T11:47:14.304 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.301+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cb9d30 req_data->id=15, curl_handle=0x55fdc4cb95e0 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cfb920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cfb920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cfb920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cfb920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cf9c80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f48c3739900 15 stack 0x55fdc4cbbce0 end 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is done 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f48c3739900 20 collect(): s=0x55fdc4bb3e20 stack=0x55fdc4cbbce0 is complete 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f48c3739900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f48c3739900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f48c3739900 15 stack 0x55fdc4bb3e20 end 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f48c3739900 20 stack->operate() returned ret=-22 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is done 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f48c3739900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f48c3739900 4 starting trim on bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cf4090:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cf4090:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f48c3739900 20 enqueued request req=0x55fdc48e7020 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f48c3739900 20 req: 0x55fdc48e7020 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is io blocked 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f4874ff1640 20 dequeued request req=0x55fdc48e7020 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f4874ff1640 20 RGWWQ: empty 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f4874ff1640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 : miss 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f4874ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f4874ff1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f4874ff1640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 info.flags=0x17 2026-03-20T11:47:14.333 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f4874ff1640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 to cache LRU end 2026-03-20T11:47:14.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f4874ff1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f4874ff1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f4874ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:47:14.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f4874ff1640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-58 : miss 2026-03-20T11:47:14.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.335+0000 7f4874ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f4874ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f4874ff1640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-58 info.flags=0x0 2026-03-20T11:47:14.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f4874ff1640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-58 to cache LRU end 2026-03-20T11:47:14.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f4874ff1640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-58 : miss 2026-03-20T11:47:14.334 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f4874ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.336 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f4874ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.336 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f4874ff1640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-58 info.flags=0x0 2026-03-20T11:47:14.336 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f4874ff1640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-58 to cache LRU end 2026-03-20T11:47:14.336 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f4874ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18): adding source pipe: {s={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f4874ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18): adding dest pipe: {s={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f4874ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f4874ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f4874ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cf4090:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cf4090:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cf4090:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cf4090:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cf9c80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cfb920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cfb920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 55bd5ac28ef6319b1150ddc688e1bc00a315201bfdd3b04ec80499a22f9619b3 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:55bd5ac28ef6319b1150ddc688e1bc00a315201bfdd3b04ec80499a22f9619b3 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 6143e1dad2cba4232972f7330f0b4ed5f17a2812c159680bb2c11d6a2ca163b0 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=6143e1dad2cba4232972f7330f0b4ed5f17a2812c159680bb2c11d6a2ca163b0 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-58%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=16, curl_handle=0x55fdc4cbda00 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is io blocked 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.336+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cbe150 req_data->id=16, curl_handle=0x55fdc4cbda00 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4cc6600:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.337 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48c3739900 15 stack 0x55fdc4d025f0 end 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48c3739900 20 run: stack=0x55fdc4d025f0 is done 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4caad20:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48c3739900 20 collect(): s=0x55fdc4ca7e70 stack=0x55fdc4d025f0 is complete 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48c3739900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48c3739900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4caad20:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48c3739900 15 stack 0x55fdc4ca7e70 end 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48c3739900 20 stack->operate() returned ret=-22 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is done 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48c3739900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48c3739900 4 starting trim on bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cc6600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cc6600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48c3739900 20 enqueued request req=0x55fdc4cc6130 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48c3739900 20 req: 0x55fdc4cc6130 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is io blocked 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48747f0640 20 dequeued request req=0x55fdc4cc6130 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48747f0640 20 RGWWQ: empty 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48747f0640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 : miss 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48747f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48747f0640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48747f0640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 info.flags=0x17 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48747f0640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 to cache LRU end 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48747f0640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48747f0640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48747f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48747f0640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-30 : miss 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.338+0000 7f48747f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48747f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48747f0640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-30 info.flags=0x0 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48747f0640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-30 to cache LRU end 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48747f0640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-30 : miss 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48747f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48747f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48747f0640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-30 info.flags=0x0 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48747f0640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-30 to cache LRU end 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48747f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35): adding source pipe: {s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48747f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35): adding dest pipe: {s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48747f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48747f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48747f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cc6600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cc6600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cc6600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.338 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cc6600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.339 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:47:14.340+0000 7f1297a1f640 -1 req 6397092041403058103 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:47:14.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4d08250:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.342 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = c549b3f7f774193a9be803d0f379847b45ccd95e910c42502767a3df980fa517 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:c549b3f7f774193a9be803d0f379847b45ccd95e910c42502767a3df980fa517 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 77dc7a8231e4632b4e72afe07c321d00e21d1eda3cf7211dc1c067b657a9fc21 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=77dc7a8231e4632b4e72afe07c321d00e21d1eda3cf7211dc1c067b657a9fc21 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-30%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=17, curl_handle=0x55fdc48e6af0 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f48c3739900 20 run: stack=0x55fdc4d025f0 is io blocked 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.339+0000 7f481e7fc640 20 link_request req_data=0x55fdc48e75f0 req_data->id=17, curl_handle=0x55fdc48e6af0 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cffd10:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cffd10:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cf7980:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f48c3739900 15 stack 0x55fdc4ca7c70 end 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f48c3739900 20 run: stack=0x55fdc4ca7c70 is done 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4d09bc0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f48c3739900 20 collect(): s=0x55fdc4bbeb40 stack=0x55fdc4ca7c70 is complete 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f48c3739900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f48c3739900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4d09bc0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f48c3739900 15 stack 0x55fdc4bbeb40 end 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f48c3739900 20 stack->operate() returned ret=-22 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is done 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f48c3739900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f48c3739900 4 starting trim on bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cc6600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cc6600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.343 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f48c3739900 20 enqueued request req=0x55fdc4cac740 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f48c3739900 20 req: 0x55fdc4cac740 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is io blocked 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f4873fef640 20 dequeued request req=0x55fdc4cac740 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f4873fef640 20 RGWWQ: empty 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f4873fef640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 : miss 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f4873fef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f4873fef640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f4873fef640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 info.flags=0x17 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f4873fef640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 to cache LRU end 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f4873fef640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f4873fef640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f4873fef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f4873fef640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-20 : miss 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f4873fef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f4873fef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f4873fef640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-20 info.flags=0x0 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f4873fef640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-20 to cache LRU end 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f4873fef640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-20 : miss 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.341+0000 7f4873fef640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cc7f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cc7f90:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4d05090:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 15 stack 0x55fdc4ca7520 end 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 run: stack=0x55fdc4ca7520 is done 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 collect(): s=0x55fdc4bc6f60 stack=0x55fdc4ca7520 is complete 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 15 stack 0x55fdc4bc6f60 end 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 stack->operate() returned ret=-22 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is done 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 4 starting trim on bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4d07a30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4d07a30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 enqueued request req=0x55fdc4cc0410 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 req: 0x55fdc4cc0410 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is io blocked 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48737ee640 20 dequeued request req=0x55fdc4cc0410 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48737ee640 20 RGWWQ: empty 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48737ee640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 : miss 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48737ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f4873fef640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f4873fef640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-20 info.flags=0x0 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f4873fef640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-20 to cache LRU end 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f4873fef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25): adding source pipe: {s={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f4873fef640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25): adding dest pipe: {s={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f4873fef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f4873fef640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f4873fef640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cc6600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cc6600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cc6600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cc6600:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.344 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cc6600:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cf4090:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cf4090:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 946e4590f61df9e9490a97cced953264458adb16ad1713b95e7cc4f4fc150b78 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:946e4590f61df9e9490a97cced953264458adb16ad1713b95e7cc4f4fc150b78 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 71ca3685f5c51c168836e6dd1ededc8d3ca596ec4b2fb251d29f4115028daa44 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=71ca3685f5c51c168836e6dd1ededc8d3ca596ec4b2fb251d29f4115028daa44 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-20%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=18, curl_handle=0x55fdc4cb95e0 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48c3739900 20 run: stack=0x55fdc4ca7520 is io blocked 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cb9d30 req_data->id=18, curl_handle=0x55fdc4cb95e0 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48737ee640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48737ee640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 info.flags=0x17 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48737ee640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 to cache LRU end 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48737ee640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48737ee640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48737ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48737ee640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-52 : miss 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.342+0000 7f48737ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48737ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48737ee640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-52 info.flags=0x0 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48737ee640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-52 to cache LRU end 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48737ee640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-52 : miss 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48737ee640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48737ee640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48737ee640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-52 info.flags=0x0 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48737ee640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-52 to cache LRU end 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48737ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12): adding source pipe: {s={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48737ee640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12): adding dest pipe: {s={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48737ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48737ee640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48737ee640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:47:14.345 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4d07a30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4d07a30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4d07a30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4d07a30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4d07a30:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4d064e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4d064e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = c63ec89b5475ac690ca7eb1684dbf30acd48ddcbe8f63a09599d3fc5726eb9ec 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:c63ec89b5475ac690ca7eb1684dbf30acd48ddcbe8f63a09599d3fc5726eb9ec 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 386aa5fedcf65888f4bbae4780859ae7c3dbcb21a96147d8fbe6e2720e44c097 2026-03-20T11:47:14.346 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=386aa5fedcf65888f4bbae4780859ae7c3dbcb21a96147d8fbe6e2720e44c097 2026-03-20T11:47:14.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-52%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=19, curl_handle=0x55fdc4cc1a40 2026-03-20T11:47:14.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f48c3739900 20 run: stack=0x55fdc4ca7c70 is io blocked 2026-03-20T11:47:14.347 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.343+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cc2150 req_data->id=19, curl_handle=0x55fdc4cc1a40 2026-03-20T11:47:14.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cfb920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cfb920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cfb920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cfb920:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cf9c80:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 15 stack 0x55fdc4cbbce0 end 2026-03-20T11:47:14.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is done 2026-03-20T11:47:14.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 20 collect(): s=0x55fdc4bb3e20 stack=0x55fdc4cbbce0 is complete 2026-03-20T11:47:14.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 10 trimming bilogs for bucket=:dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) markers=[,00000000001.98.6,,,,00000000001.102.6,,,00000000001.40.6,,00000000001.48.6], shards=11 2026-03-20T11:47:14.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cf9c80:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 10 trimming bilog shard 1 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.98.6 2026-03-20T11:47:14.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 10 trimming bilog shard 5 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.102.6 2026-03-20T11:47:14.377 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 10 trimming bilog shard 8 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.40.6 2026-03-20T11:47:14.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 10 trimming bilog shard 10 of :dipepu-58[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18]) at marker 00000000001.48.6 2026-03-20T11:47:14.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4d05cb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cc48f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cfd190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc482f600:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4d05cb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.378 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.1 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is io blocked 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cc48f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.5 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 20 run: stack=0x55fdc4cf64d0 is io blocked 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cfd190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.8 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 20 run: stack=0x55fdc4d01f90 is io blocked 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc482f600:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18.10 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.380+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is io blocked 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4d08250:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f48c3739900 15 stack 0x55fdc4d025f0 end 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f48c3739900 20 run: stack=0x55fdc4d025f0 is done 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f48c3739900 20 collect(): s=0x55fdc4ca7e70 stack=0x55fdc4d025f0 is complete 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f48c3739900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f48c3739900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f48c3739900 15 stack 0x55fdc4ca7e70 end 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f48c3739900 20 stack->operate() returned ret=-22 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is done 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f48c3739900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f48c3739900 4 starting trim on bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cb1a00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cb1a00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f48c3739900 20 enqueued request req=0x55fdc4cc6130 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.379 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f48c3739900 20 req: 0x55fdc4cc6130 2026-03-20T11:47:14.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is io blocked 2026-03-20T11:47:14.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f4872fed640 20 dequeued request req=0x55fdc4cc6130 2026-03-20T11:47:14.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f4872fed640 20 RGWWQ: empty 2026-03-20T11:47:14.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f4872fed640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 : miss 2026-03-20T11:47:14.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f4872fed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f4872fed640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:47:14.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f4872fed640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 info.flags=0x17 2026-03-20T11:47:14.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f4872fed640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 to cache LRU end 2026-03-20T11:47:14.380 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f4872fed640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f4872fed640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f4872fed640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f4872fed640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-42 : miss 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.381+0000 7f4872fed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cfd190:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cfd190:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 15 stack 0x55fdc4d01f90 end 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 run: stack=0x55fdc4d01f90 is done 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cf9c80:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc482f600:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc482f600:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 15 stack 0x55fdc4cf32a0 end 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is done 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cf9c80:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f4872fed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f4872fed640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-42 info.flags=0x0 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f4872fed640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-42 to cache LRU end 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f4872fed640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-42 : miss 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f4872fed640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cc48f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cc48f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 15 stack 0x55fdc4cf64d0 end 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 run: stack=0x55fdc4cf64d0 is done 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cf9c80:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f4872fed640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f4872fed640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-42 info.flags=0x0 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f4872fed640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-42 to cache LRU end 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f4872fed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2): adding source pipe: {s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f4872fed640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2): adding dest pipe: {s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f4872fed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f4872fed640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f4872fed640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4d05cb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4d05cb0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 15 stack 0x55fdc4cbbce0 end 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is done 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cf9c80:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cb1a00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 trim: trimmed bucket instance dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 15 stack 0x55fdc4bb3e20 end 2026-03-20T11:47:14.381 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is done 2026-03-20T11:47:14.381 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:47:14.383+0000 7f121511a640 -1 req 4476650543536893416 0.001000002s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:47:14.382 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cb1a00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.382 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.382 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.382 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cb1a00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.382 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.382 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 4 starting trim on bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:47:14.382 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cb1a00:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4d05cb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4d05cb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 enqueued request req=0x55fdc4cc6130 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 req: 0x55fdc4cc6130 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48797fa640 20 dequeued request req=0x55fdc4cc6130 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48797fa640 20 RGWWQ: empty 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48797fa640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 : miss 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48797fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is io blocked 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cfc150:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4d081c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4d081c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 98bdb5ad01c270869200f292069d3b70f2e12c54b229946bad63a0459d549977 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:98bdb5ad01c270869200f292069d3b70f2e12c54b229946bad63a0459d549977 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 83ff17491aa0e1c8f7bb3711c051dfd451e2a3105b4736cf4a7f0ad15c15d06d 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=83ff17491aa0e1c8f7bb3711c051dfd451e2a3105b4736cf4a7f0ad15c15d06d 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-42%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=20, curl_handle=0x55fdc48e6af0 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is io blocked 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f481e7fc640 20 link_request req_data=0x55fdc48e75f0 req_data->id=20, curl_handle=0x55fdc48e6af0 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48797fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48797fa640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 info.flags=0x17 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48797fa640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 to cache LRU end 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48797fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.383 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48797fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.384 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:47:14.384+0000 7f129621c640 -1 req 2856838018507215999 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48797fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48797fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-2 : miss 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.382+0000 7f48797fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48797fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48797fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-2 info.flags=0x0 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48797fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-2 to cache LRU end 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48797fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-2 : miss 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48797fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48797fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48797fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-2 info.flags=0x0 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48797fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-2 to cache LRU end 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48797fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2): adding source pipe: {s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48797fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2): adding dest pipe: {s={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48797fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48797fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48797fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4d05cb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4d05cb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4d05cb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4d05cb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4d05cb0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cfd190:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cfd190:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.385 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 74b5d8788826edcd2dd8e4827d10b13f5142bb36e520667681572fcaa7552079 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:74b5d8788826edcd2dd8e4827d10b13f5142bb36e520667681572fcaa7552079 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 21ea0281db1d6b0571cd766f49d41045cf9650848a7875b6ecbefc4d7a1eb000 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=21ea0281db1d6b0571cd766f49d41045cf9650848a7875b6ecbefc4d7a1eb000 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-2%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=21, curl_handle=0x55fdc4cbda00 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f48c3739900 20 run: stack=0x55fdc4cf64d0 is io blocked 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.383+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cbe150 req_data->id=21, curl_handle=0x55fdc4cbda00 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.384+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cf4090:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.384+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cf4090:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.384+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cf4090:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.384+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cf4090:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.384+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cc6600:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.384+0000 7f48c3739900 15 stack 0x55fdc4ca7520 end 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.384+0000 7f48c3739900 20 run: stack=0x55fdc4ca7520 is done 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.384+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.384+0000 7f48c3739900 20 collect(): s=0x55fdc4bbeb40 stack=0x55fdc4ca7520 is complete 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.384+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.384+0000 7f48c3739900 10 trimming bilogs for bucket=:dipepu-20[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25]) markers=[,,,,,,,,00000000001.41.14,,], shards=11 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.384+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cf4090:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.384+0000 7f48c3739900 10 trimming bilog shard 8 of :dipepu-20[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25]) at marker 00000000001.41.14 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.384+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.384+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cf6b90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.384+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cf6b90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.384+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25.8 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.384+0000 7f48c3739900 20 run: stack=0x55fdc4ca7520 is io blocked 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.385+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cf6b90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.385+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cf6b90:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.385+0000 7f48c3739900 15 stack 0x55fdc4ca7520 end 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.385+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.385+0000 7f48c3739900 20 run: stack=0x55fdc4ca7520 is done 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.385+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cf4090:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.385+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.385+0000 7f48c3739900 20 trim: trimmed bucket instance dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.385+0000 7f48c3739900 15 stack 0x55fdc4bbeb40 end 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.385+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is done 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.385+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.385+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.386 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.385+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.385+0000 7f48c3739900 4 starting trim on bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.385+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cf6b90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.385+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cf6b90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.385+0000 7f48c3739900 20 enqueued request req=0x55fdc4cac740 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.385+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.385+0000 7f48c3739900 20 req: 0x55fdc4cac740 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.385+0000 7f48727ec640 20 dequeued request req=0x55fdc4cac740 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.385+0000 7f48727ec640 20 RGWWQ: empty 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.385+0000 7f48727ec640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 : miss 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.385+0000 7f48727ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.385+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is io blocked 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48727ec640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48727ec640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 info.flags=0x17 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48727ec640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 to cache LRU end 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48727ec640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48727ec640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48727ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48727ec640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-57 : miss 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48727ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48727ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48727ec640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-57 info.flags=0x0 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48727ec640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-57 to cache LRU end 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48727ec640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-57 : miss 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48727ec640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48727ec640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48727ec640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-57 info.flags=0x0 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48727ec640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-57 to cache LRU end 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48727ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17): adding source pipe: {s={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48727ec640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17): adding dest pipe: {s={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48727ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48727ec640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48727ec640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cf6b90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cf6b90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cf6b90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cf6b90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cf4090:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cc6600:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cc6600:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.387 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 697bd71f66e51322b61d53e10bc7fe55a4ed9b248014aec72ed97423fe37d9fa 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:697bd71f66e51322b61d53e10bc7fe55a4ed9b248014aec72ed97423fe37d9fa 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 18005f0b8a7f232b891e081cd6df78089583b3551bfd6dfa4a945be7235f9569 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=18005f0b8a7f232b891e081cd6df78089583b3551bfd6dfa4a945be7235f9569 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-57%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=22, curl_handle=0x55fdc4cb95e0 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f48c3739900 20 run: stack=0x55fdc4ca7520 is io blocked 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.386+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cb9d30 req_data->id=22, curl_handle=0x55fdc4cb95e0 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4d064e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4d064e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4d064e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4d064e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4d07a30:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 15 stack 0x55fdc4ca7c70 end 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 20 run: stack=0x55fdc4ca7c70 is done 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 20 collect(): s=0x55fdc4bc6f60 stack=0x55fdc4ca7c70 is complete 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 10 trimming bilogs for bucket=:dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) markers=[00000000002.30.6,,00000000002.60.6,,00000000002.47.6,,,00000000002.30.6,,,], shards=11 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4d064e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 10 trimming bilog shard 0 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.30.6 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 10 trimming bilog shard 2 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.60.6 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 10 trimming bilog shard 4 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.47.6 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 10 trimming bilog shard 7 of :dipepu-52[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12]) at marker 00000000002.30.6 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cf93f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc48302d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cff2f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4cfdf30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cf93f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.0 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 20 run: stack=0x55fdc4ca7c70 is io blocked 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc48302d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.2 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is io blocked 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cff2f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.4 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 20 run: stack=0x55fdc4d01f90 is io blocked 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4cfdf30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12.7 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.387+0000 7f48c3739900 20 run: stack=0x55fdc4d025f0 is io blocked 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.388+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4cfdf30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.388+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4cfdf30:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.388+0000 7f48c3739900 15 stack 0x55fdc4d025f0 end 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.388+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.388+0000 7f48c3739900 20 run: stack=0x55fdc4d025f0 is done 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.388+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4d064e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.388 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.388+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cf93f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cf93f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 15 stack 0x55fdc4ca7c70 end 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 run: stack=0x55fdc4ca7c70 is done 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4d064e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc48302d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc48302d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 15 stack 0x55fdc4cf32a0 end 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is done 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4d064e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cff2f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cff2f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 15 stack 0x55fdc4d01f90 end 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 run: stack=0x55fdc4d01f90 is done 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4d064e0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 trim: trimmed bucket instance dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 15 stack 0x55fdc4bc6f60 end 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is done 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 4 starting trim on bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4d00880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4d00880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 enqueued request req=0x55fdc4ca84b0 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 req: 0x55fdc4ca84b0 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f4871feb640 20 dequeued request req=0x55fdc4ca84b0 2026-03-20T11:47:14.389 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f4871feb640 20 RGWWQ: empty 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f4871feb640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 : miss 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f4871feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is io blocked 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f4871feb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f4871feb640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 info.flags=0x17 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f4871feb640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 to cache LRU end 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f4871feb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f4871feb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f4871feb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f4871feb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-29 : miss 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f4871feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f4871feb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f4871feb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-29 info.flags=0x0 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f4871feb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-29 to cache LRU end 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f4871feb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-29 : miss 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.389+0000 7f4871feb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f4871feb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f4871feb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-29 info.flags=0x0 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f4871feb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-29 to cache LRU end 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f4871feb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34): adding source pipe: {s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f4871feb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34): adding dest pipe: {s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f4871feb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f4871feb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f4871feb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4d00880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4d00880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4d00880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4d00880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4d064e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4d01100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4d01100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = b76cd04a1f63f1f3ac534d4393c620b89ad2e124d99ed0e80a2b304d4f7e0df0 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:b76cd04a1f63f1f3ac534d4393c620b89ad2e124d99ed0e80a2b304d4f7e0df0 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.390 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 6e44e7ed80f401f21eb10747d090b001c82d201047a9c412f5e3949086022a13 2026-03-20T11:47:14.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=6e44e7ed80f401f21eb10747d090b001c82d201047a9c412f5e3949086022a13 2026-03-20T11:47:14.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-29%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=23, curl_handle=0x55fdc4cc1a40 2026-03-20T11:47:14.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f48c3739900 20 run: stack=0x55fdc4d01f90 is io blocked 2026-03-20T11:47:14.391 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.390+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cc2150 req_data->id=23, curl_handle=0x55fdc4cc1a40 2026-03-20T11:47:14.391 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:47:14.390+0000 7f125799f640 -1 req 12620065132616302742 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:47:14.421 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4d081c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.421 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4d081c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:14.421 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cfc150:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.421 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48c3739900 15 stack 0x55fdc4cbbce0 end 2026-03-20T11:47:14.421 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is done 2026-03-20T11:47:14.421 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.421 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48c3739900 20 collect(): s=0x55fdc4ca7e70 stack=0x55fdc4cbbce0 is complete 2026-03-20T11:47:14.421 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.421 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48c3739900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:14.421 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48c3739900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:14.421 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:14.421 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48c3739900 15 stack 0x55fdc4ca7e70 end 2026-03-20T11:47:14.421 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48c3739900 20 stack->operate() returned ret=-22 2026-03-20T11:47:14.421 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is done 2026-03-20T11:47:14.421 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.421 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48c3739900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:14.421 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.421 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.421 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48c3739900 4 starting trim on bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:47:14.421 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d00880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.421 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d00880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.421 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48c3739900 20 enqueued request req=0x55fdc48e7020 2026-03-20T11:47:14.421 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.421 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48c3739900 20 req: 0x55fdc48e7020 2026-03-20T11:47:14.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48717ea640 20 dequeued request req=0x55fdc48e7020 2026-03-20T11:47:14.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48717ea640 20 RGWWQ: empty 2026-03-20T11:47:14.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48717ea640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 : miss 2026-03-20T11:47:14.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48717ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is io blocked 2026-03-20T11:47:14.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48717ea640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48717ea640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 info.flags=0x17 2026-03-20T11:47:14.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48717ea640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 to cache LRU end 2026-03-20T11:47:14.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48717ea640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.422 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48717ea640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48717ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48717ea640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-23 : miss 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.424+0000 7f48717ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cfd190:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cfd190:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4d05cb0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 15 stack 0x55fdc4cf64d0 end 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 run: stack=0x55fdc4cf64d0 is done 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 collect(): s=0x55fdc4bb3e20 stack=0x55fdc4cf64d0 is complete 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 15 stack 0x55fdc4bb3e20 end 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 stack->operate() returned ret=-22 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is done 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 4 starting trim on bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4d05cb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4d05cb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 enqueued request req=0x55fdc4cc6130 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 req: 0x55fdc4cc6130 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f4870fe9640 20 dequeued request req=0x55fdc4cc6130 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f4870fe9640 20 RGWWQ: empty 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f4870fe9640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 : miss 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f4870fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48717ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48717ea640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-23 info.flags=0x0 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48717ea640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-23 to cache LRU end 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48717ea640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-23 : miss 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48717ea640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is io blocked 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48717ea640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48717ea640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-23 info.flags=0x0 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48717ea640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-23 to cache LRU end 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48717ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28): adding source pipe: {s={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48717ea640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28): adding dest pipe: {s={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.423 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48717ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48717ea640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48717ea640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f4870fe9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f4870fe9640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 info.flags=0x17 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f4870fe9640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 to cache LRU end 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f4870fe9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f4870fe9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f4870fe9640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f4870fe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f4870fe9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-34 : miss 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f4870fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d00880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d00880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d00880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d00880:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cfc150:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4d081c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4d081c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 98c317eeb4384b30c7c02a55327fbdfc0e003958e183daebc91749c29b3a11d4 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:98c317eeb4384b30c7c02a55327fbdfc0e003958e183daebc91749c29b3a11d4 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = cbc7e063067eb3a59f7249c5aadfc00c67919b57a7b586c26b1a472f04fee8e1 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=cbc7e063067eb3a59f7249c5aadfc00c67919b57a7b586c26b1a472f04fee8e1 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.424 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-23%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=24, curl_handle=0x55fdc4cbda00 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f48c3739900 20 run: stack=0x55fdc4cf64d0 is io blocked 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cbe150 req_data->id=24, curl_handle=0x55fdc4cbda00 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.425+0000 7f4870fe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.426+0000 7f4870fe9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-34 info.flags=0x0 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.426+0000 7f4870fe9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-34 to cache LRU end 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.426+0000 7f4870fe9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-34 : miss 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.426+0000 7f4870fe9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f4870fe9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f4870fe9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-34 info.flags=0x0 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f4870fe9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-34 to cache LRU end 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f4870fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39): adding source pipe: {s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f4870fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39): adding dest pipe: {s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f4870fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f4870fe9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f4870fe9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4d05cb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4d05cb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4d05cb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4d05cb0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4d07a30:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4d00880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4d00880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.425 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = e6d04aa1ab8f87fa54ca578da54fa45d7e8819cd00d745342ce8bdea30c61ca9 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr:e6d04aa1ab8f87fa54ca578da54fa45d7e8819cd00d745342ce8bdea30c61ca9 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 0be2deec625635ca511f78c8e0c3babfc9de60701dad007ab685dec687dd83fb 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=0be2deec625635ca511f78c8e0c3babfc9de60701dad007ab685dec687dd83fb 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-34%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=25, curl_handle=0x55fdc48e6af0 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is io blocked 2026-03-20T11:47:14.426 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.427+0000 7f481e7fc640 20 link_request req_data=0x55fdc48e75f0 req_data->id=25, curl_handle=0x55fdc48e6af0 2026-03-20T11:47:14.426 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:47:14.427+0000 7f1208100640 -1 req 3445863176676480832 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.429+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cc6600:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.429+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cc6600:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.429+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cc6600:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.429+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cc6600:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.429+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cf4090:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.429+0000 7f48c3739900 15 stack 0x55fdc4ca7520 end 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.429+0000 7f48c3739900 20 run: stack=0x55fdc4ca7520 is done 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.429+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.429+0000 7f48c3739900 20 collect(): s=0x55fdc4bbeb40 stack=0x55fdc4ca7520 is complete 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.429+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.429+0000 7f48c3739900 10 trimming bilogs for bucket=:dipepu-57[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17]) markers=[,00000000001.63.6,,,,00000000001.43.6,,,,,], shards=11 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.429+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cae8a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.429+0000 7f48c3739900 10 trimming bilog shard 1 of :dipepu-57[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17]) at marker 00000000001.63.6 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.429+0000 7f48c3739900 10 trimming bilog shard 5 of :dipepu-57[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17]) at marker 00000000001.43.6 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.429+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.429+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4d05cb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.429+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cf6b90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.429+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4d05cb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.429+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17.1 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.429+0000 7f48c3739900 20 run: stack=0x55fdc4ca7520 is io blocked 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.429+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cf6b90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.429+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17.5 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.429+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is io blocked 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4d05cb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4d05cb0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48c3739900 15 stack 0x55fdc4ca7520 end 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48c3739900 20 run: stack=0x55fdc4ca7520 is done 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cae8a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.427 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.428 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cf6b90:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.428 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cf6b90:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.428 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48c3739900 15 stack 0x55fdc4cf32a0 end 2026-03-20T11:47:14.428 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.428 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is done 2026-03-20T11:47:14.428 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cae8a0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.428 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.428 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48c3739900 20 trim: trimmed bucket instance dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17 2026-03-20T11:47:14.428 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48c3739900 15 stack 0x55fdc4bbeb40 end 2026-03-20T11:47:14.428 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is done 2026-03-20T11:47:14.428 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.428 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48c3739900 4 starting trim on bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc482c6e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc482c6e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48c3739900 20 enqueued request req=0x55fdc4d03480 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48c3739900 20 req: 0x55fdc4d03480 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48707e8640 20 dequeued request req=0x55fdc4d03480 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48707e8640 20 RGWWQ: empty 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48707e8640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 : miss 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48707e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is io blocked 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48707e8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48707e8640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 info.flags=0x17 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48707e8640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 to cache LRU end 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48707e8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48707e8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.430+0000 7f48707e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48707e8640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-54 : miss 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48707e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4d01100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4d01100:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4d064e0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 15 stack 0x55fdc4d01f90 end 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 run: stack=0x55fdc4d01f90 is done 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 collect(): s=0x55fdc4bc6f60 stack=0x55fdc4d01f90 is complete 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 15 stack 0x55fdc4bc6f60 end 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 stack->operate() returned ret=-22 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is done 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 4 starting trim on bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cf6b90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cf6b90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 enqueued request req=0x55fdc4ca84b0 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 req: 0x55fdc4ca84b0 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f486ffe7640 20 dequeued request req=0x55fdc4ca84b0 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f486ffe7640 20 RGWWQ: empty 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f486ffe7640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 : miss 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f486ffe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is io blocked 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f486ffe7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f486ffe7640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 info.flags=0x17 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f486ffe7640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 to cache LRU end 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f486ffe7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f486ffe7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f486ffe7640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f486ffe7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f486ffe7640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-32 : miss 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f486ffe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48707e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48707e8640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-54 info.flags=0x0 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48707e8640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-54 to cache LRU end 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48707e8640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-54 : miss 2026-03-20T11:47:14.429 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48707e8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f486ffe7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f486ffe7640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-32 info.flags=0x0 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f486ffe7640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-32 to cache LRU end 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f486ffe7640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-32 : miss 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f486ffe7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48707e8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48707e8640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-54 info.flags=0x0 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48707e8640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-54 to cache LRU end 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48707e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14): adding source pipe: {s={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48707e8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14): adding dest pipe: {s={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48707e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48707e8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48707e8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc482c6e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc482c6e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc482c6e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc482c6e0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cf4090:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cc6600:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cc6600:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 8a359311a6dd924e7425a2ca8742880205d91cc4465129a9ca9d77ea8b1b3569 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:8a359311a6dd924e7425a2ca8742880205d91cc4465129a9ca9d77ea8b1b3569 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.430 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = c9bd25d9c8b95960357039f210b1a17b5b83d4f41e0a9e5204ff1d65c65f4343 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c9bd25d9c8b95960357039f210b1a17b5b83d4f41e0a9e5204ff1d65c65f4343 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-54%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=26, curl_handle=0x55fdc4cc1a40 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f48c3739900 20 run: stack=0x55fdc4d01f90 is io blocked 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.431+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cc2150 req_data->id=26, curl_handle=0x55fdc4cc1a40 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f486ffe7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f486ffe7640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-32 info.flags=0x0 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f486ffe7640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-32 to cache LRU end 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f486ffe7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37): adding source pipe: {s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f486ffe7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37): adding dest pipe: {s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f486ffe7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f486ffe7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f486ffe7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cf6b90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cf6b90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cf6b90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cf6b90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4d01100:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc48302d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc48302d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.431 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 51095b0e258766a75e030cc6a589a9265292e5b27ea26edf0052ba424d1922bf 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:51095b0e258766a75e030cc6a589a9265292e5b27ea26edf0052ba424d1922bf 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = f32583d46a4e3dee704395445feb602df2dab410a7405a4c6e4fa4853ada1846 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=f32583d46a4e3dee704395445feb602df2dab410a7405a4c6e4fa4853ada1846 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-32%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=27, curl_handle=0x55fdc4cb95e0 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is io blocked 2026-03-20T11:47:14.432 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.432+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cb9d30 req_data->id=27, curl_handle=0x55fdc4cb95e0 2026-03-20T11:47:14.432 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:47:14.433+0000 7f1288a01640 -1 req 12305478794119767811 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:47:14.465 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4d081c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.465 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4d081c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.465 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4d00880:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.465 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4d00880:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:14.465 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4d081c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.465 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4d07a30:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.465 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 15 stack 0x55fdc4cbbce0 end 2026-03-20T11:47:14.465 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is done 2026-03-20T11:47:14.465 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4d081c0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.465 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.465 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 collect(): s=0x55fdc4bb3e20 stack=0x55fdc4cbbce0 is complete 2026-03-20T11:47:14.465 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 15 stack 0x55fdc4bb3e20 end 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 stack->operate() returned ret=-22 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is done 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cfc150:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 15 stack 0x55fdc4cf64d0 end 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 run: stack=0x55fdc4cf64d0 is done 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 collect(): s=0x55fdc4ca7e70 stack=0x55fdc4cf64d0 is complete 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 10 trimming bilogs for bucket=:dipepu-23[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28]) markers=[,,,,,,,,00000000002.27.13,,], shards=11 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 4 starting trim on bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc48299b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 10 trimming bilog shard 8 of :dipepu-23[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28]) at marker 00000000002.27.13 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4d07a30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4d05cb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4d07a30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 enqueued request req=0x55fdc4cc0410 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 req: 0x55fdc4cc0410 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f486f7e6640 20 dequeued request req=0x55fdc4cc0410 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f486f7e6640 20 RGWWQ: empty 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f486f7e6640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 : miss 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f486f7e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is io blocked 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4d05cb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28.8 2026-03-20T11:47:14.466 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f48c3739900 20 run: stack=0x55fdc4cf64d0 is io blocked 2026-03-20T11:47:14.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f486f7e6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f486f7e6640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 info.flags=0x17 2026-03-20T11:47:14.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f486f7e6640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 to cache LRU end 2026-03-20T11:47:14.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f486f7e6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f486f7e6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f486f7e6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:47:14.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.468+0000 7f486f7e6640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-19 : miss 2026-03-20T11:47:14.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f486f7e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f486f7e6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f486f7e6640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-19 info.flags=0x0 2026-03-20T11:47:14.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f486f7e6640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-19 to cache LRU end 2026-03-20T11:47:14.467 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f486f7e6640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-19 : miss 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f486f7e6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f486f7e6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f486f7e6640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-19 info.flags=0x0 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f486f7e6640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-19 to cache LRU end 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f486f7e6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24): adding source pipe: {s={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f486f7e6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24): adding dest pipe: {s={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f486f7e6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f486f7e6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f486f7e6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4d07a30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4d07a30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4d07a30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4d07a30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cab160:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = ce2845f48a32217aec47e3498bb3d4067e9eb28ed9fe30a0fd9ca58b57e24ee3 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:ce2845f48a32217aec47e3498bb3d4067e9eb28ed9fe30a0fd9ca58b57e24ee3 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 8568daf9164e65c6bbe20956792071e39cd67b3e9d024bea73200ed007a8f6b7 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=8568daf9164e65c6bbe20956792071e39cd67b3e9d024bea73200ed007a8f6b7 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-19%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.468 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=28, curl_handle=0x55fdc48e6af0 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is io blocked 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4d05cb0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4d05cb0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 15 stack 0x55fdc4cf64d0 end 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 run: stack=0x55fdc4cf64d0 is done 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc48299b0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 trim: trimmed bucket instance dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 15 stack 0x55fdc4ca7e70 end 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is done 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 4 starting trim on bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d07a30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d07a30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 enqueued request req=0x55fdc4cac740 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 req: 0x55fdc4cac740 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f486efe5640 20 dequeued request req=0x55fdc4cac740 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f486efe5640 20 RGWWQ: empty 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f486efe5640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 : miss 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f486efe5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.469+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is io blocked 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f481e7fc640 20 link_request req_data=0x55fdc48e75f0 req_data->id=28, curl_handle=0x55fdc48e6af0 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f486efe5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f486efe5640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 info.flags=0x17 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f486efe5640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 to cache LRU end 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f486efe5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f486efe5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f486efe5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f486efe5640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-4 : miss 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f486efe5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f486efe5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f486efe5640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-4 info.flags=0x0 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f486efe5640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-4 to cache LRU end 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f486efe5640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-4 : miss 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f486efe5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f486efe5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f486efe5640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-4 info.flags=0x0 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f486efe5640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-4 to cache LRU end 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f486efe5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5): adding source pipe: {s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f486efe5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5): adding dest pipe: {s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f486efe5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f486efe5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f486efe5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d07a30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d07a30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d07a30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d07a30:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4d081c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.469 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4d064e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4d064e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = ecba1afea3ca3178343e14c8b7408b70b6f6a1fb5846ce5fc4429f283e1bf5f2 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:ecba1afea3ca3178343e14c8b7408b70b6f6a1fb5846ce5fc4429f283e1bf5f2 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 79ddf2f2a2c348d0f22762f2a29e67b52b854a27c1186430bf7685bad0179854 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=79ddf2f2a2c348d0f22762f2a29e67b52b854a27c1186430bf7685bad0179854 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-4%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=29, curl_handle=0x55fdc4cbda00 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f48c3739900 20 run: stack=0x55fdc4cf64d0 is io blocked 2026-03-20T11:47:14.470 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.470+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cbe150 req_data->id=29, curl_handle=0x55fdc4cbda00 2026-03-20T11:47:14.471 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:47:14.471+0000 7f12591a2640 -1 req 503303803491709149 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:47:14.472 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cc6600:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.472 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cc6600:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.472 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc48302d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.472 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc48302d0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:14.472 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cc6600:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.472 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4d01100:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.472 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 15 stack 0x55fdc4cf32a0 end 2026-03-20T11:47:14.472 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is done 2026-03-20T11:47:14.472 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cc6600:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.472 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.472 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 collect(): s=0x55fdc4bc6f60 stack=0x55fdc4cf32a0 is complete 2026-03-20T11:47:14.472 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.472 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:14.472 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:14.472 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:14.472 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 15 stack 0x55fdc4bc6f60 end 2026-03-20T11:47:14.472 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 stack->operate() returned ret=-22 2026-03-20T11:47:14.472 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is done 2026-03-20T11:47:14.472 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cf4090:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.472 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 15 stack 0x55fdc4d01f90 end 2026-03-20T11:47:14.472 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 run: stack=0x55fdc4d01f90 is done 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 collect(): s=0x55fdc4bbeb40 stack=0x55fdc4d01f90 is complete 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 10 trimming bilogs for bucket=:dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) markers=[00000000004.65.1,00000000002.57.1,00000000004.95.1,00000000002.36.1,00000000004.41.1,00000000002.67.1,00000000002.53.1,00000000004.82.1,00000000002.67.1,00000000002.123.1,00000000002.39.1], shards=11 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 4 starting trim on bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cf4090:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 10 trimming bilog shard 0 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.65.1 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 10 trimming bilog shard 1 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.57.1 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 10 trimming bilog shard 2 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.95.1 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 10 trimming bilog shard 3 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.36.1 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 10 trimming bilog shard 4 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.41.1 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 10 trimming bilog shard 5 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.67.1 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 10 trimming bilog shard 6 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.53.1 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 10 trimming bilog shard 7 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000004.82.1 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 10 trimming bilog shard 8 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.67.1 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 10 trimming bilog shard 9 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.123.1 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 10 trimming bilog shard 10 of :dipepu-54[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14]) at marker 00000000002.39.1 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4829fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc482dba0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc48314c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cc48a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cff210:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4caf430:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc482f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4cb4790:op=0x55fdc4cb2890:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4cfb2d0:op=0x55fdc4cb0580:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4cb8810:op=0x55fdc482b530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4cfe920:op=0x55fdc482bd30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc482fea0:op=0x55fdc48302d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4829fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 enqueued request req=0x55fdc4cfeb50 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 req: 0x55fdc4cfeb50 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is io blocked 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc482dba0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.473 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.0 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48867fc640 20 dequeued request req=0x55fdc4cfeb50 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48867fc640 20 RGWWQ: empty 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48867fc640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 : miss 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48867fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 run: stack=0x55fdc4d01f90 is io blocked 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc48314c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.1 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is io blocked 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cc48a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.2 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 run: stack=0x55fdc4ca7520 is io blocked 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cff210:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.3 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 run: stack=0x55fdc4ca7c70 is io blocked 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4caf430:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.4 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 run: stack=0x55fdc4d025f0 is io blocked 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc482f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.5 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is io blocked 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4cb4790:op=0x55fdc4cb2890:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.6 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 run: stack=0x55fdc4cb4790 is io blocked 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4cfb2d0:op=0x55fdc4cb0580:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.7 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 run: stack=0x55fdc4cfb2d0 is io blocked 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4cb8810:op=0x55fdc482b530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.8 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 run: stack=0x55fdc4cb8810 is io blocked 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc4cfe920:op=0x55fdc482bd30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.9 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 run: stack=0x55fdc4cfe920 is io blocked 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 cr:s=0x55fdc482fea0:op=0x55fdc48302d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14.10 2026-03-20T11:47:14.474 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.474+0000 7f48c3739900 20 run: stack=0x55fdc482fea0 is io blocked 2026-03-20T11:47:14.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.475+0000 7f48867fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.475+0000 7f48867fc640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 info.flags=0x17 2026-03-20T11:47:14.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.475+0000 7f48867fc640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 to cache LRU end 2026-03-20T11:47:14.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.475+0000 7f48867fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.475+0000 7f48867fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.475+0000 7f48867fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:47:14.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.475+0000 7f48867fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-53 : miss 2026-03-20T11:47:14.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48867fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc48314c0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc48314c0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 15 stack 0x55fdc4cf32a0 end 2026-03-20T11:47:14.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is done 2026-03-20T11:47:14.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cf4090:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48867fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.475 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48867fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-53 info.flags=0x0 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48867fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-53 to cache LRU end 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48867fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-53 : miss 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48867fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cff210:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cff210:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 15 stack 0x55fdc4ca7c70 end 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 run: stack=0x55fdc4ca7c70 is done 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cf4090:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cc48a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cc48a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 15 stack 0x55fdc4ca7520 end 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 run: stack=0x55fdc4ca7520 is done 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cf4090:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4caf430:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4caf430:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 15 stack 0x55fdc4d025f0 end 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 run: stack=0x55fdc4d025f0 is done 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cf4090:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48867fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48867fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-53 info.flags=0x0 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48867fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-53 to cache LRU end 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48867fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13): adding source pipe: {s={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48867fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13): adding dest pipe: {s={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48867fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48867fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48867fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4829fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4829fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4829fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4829fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4cc6600:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.476 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4d01100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4d01100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 8a469d0c210a8bafa4e2ffe1accbf5a5b268e500bcf0d854779950d8dff3efa0 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:8a469d0c210a8bafa4e2ffe1accbf5a5b268e500bcf0d854779950d8dff3efa0 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = cbc14e01fea0452ba6e3a51373909cf111d44979035041f684ec2b526ef9fc4c 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=cbc14e01fea0452ba6e3a51373909cf111d44979035041f684ec2b526ef9fc4c 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-53%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=30, curl_handle=0x55fdc4cb95e0 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 run: stack=0x55fdc4d025f0 is io blocked 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 cr:s=0x55fdc482fea0:op=0x55fdc48302d0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 cr:s=0x55fdc482fea0:op=0x55fdc48302d0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 15 stack 0x55fdc482fea0 end 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 run: stack=0x55fdc482fea0 is done 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cf4090:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.477 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cb9d30 req_data->id=30, curl_handle=0x55fdc4cb95e0 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc482f1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc482f1f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 15 stack 0x55fdc48e4c10 end 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is done 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cf4090:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.476+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 cr:s=0x55fdc4cb8810:op=0x55fdc482b530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 cr:s=0x55fdc4cb8810:op=0x55fdc482b530:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 15 stack 0x55fdc4cb8810 end 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 run: stack=0x55fdc4cb8810 is done 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cf4090:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 cr:s=0x55fdc4cfb2d0:op=0x55fdc4cb0580:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 cr:s=0x55fdc4cfb2d0:op=0x55fdc4cb0580:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 15 stack 0x55fdc4cfb2d0 end 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 run: stack=0x55fdc4cfb2d0 is done 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc482dba0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc482dba0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 15 stack 0x55fdc4d01f90 end 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 run: stack=0x55fdc4d01f90 is done 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 cr:s=0x55fdc4cb4790:op=0x55fdc4cb2890:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 cr:s=0x55fdc4cb4790:op=0x55fdc4cb2890:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 15 stack 0x55fdc4cb4790 end 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 run: stack=0x55fdc4cb4790 is done 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cf4090:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cf4090:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cf4090:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 cr:s=0x55fdc4cfe920:op=0x55fdc482bd30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 cr:s=0x55fdc4cfe920:op=0x55fdc482bd30:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 15 stack 0x55fdc4cfe920 end 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 run: stack=0x55fdc4cfe920 is done 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4cf4090:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 cr:s=0x55fdc4bbeb40:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 trim: trimmed bucket instance dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 15 stack 0x55fdc4bbeb40 end 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 run: stack=0x55fdc4bbeb40 is done 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 cr:s=0x55fdc4cfb2d0:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 4 starting trim on bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 cr:s=0x55fdc4cfb2d0:op=0x55fdc4829fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 cr:s=0x55fdc4cfb2d0:op=0x55fdc4829fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 enqueued request req=0x55fdc482aa80 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 req: 0x55fdc482aa80 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f486e7e4640 20 dequeued request req=0x55fdc482aa80 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f486e7e4640 20 RGWWQ: empty 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f486e7e4640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 : miss 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f486e7e4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.478 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f48c3739900 20 run: stack=0x55fdc4cfb2d0 is io blocked 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f486e7e4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f486e7e4640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 info.flags=0x17 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f486e7e4640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 to cache LRU end 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f486e7e4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f486e7e4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f486e7e4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f486e7e4640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-14 : miss 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.478+0000 7f486e7e4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f486e7e4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f486e7e4640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-14 info.flags=0x0 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f486e7e4640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-14 to cache LRU end 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f486e7e4640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-14 : miss 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f486e7e4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f486e7e4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f486e7e4640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-14 info.flags=0x0 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f486e7e4640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-14 to cache LRU end 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f486e7e4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19): adding source pipe: {s={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f486e7e4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19): adding dest pipe: {s={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f486e7e4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f486e7e4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f486e7e4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 20 cr:s=0x55fdc4cfb2d0:op=0x55fdc4829fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 20 cr:s=0x55fdc4cfb2d0:op=0x55fdc4829fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 20 cr:s=0x55fdc4cfb2d0:op=0x55fdc4829fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 20 cr:s=0x55fdc4cfb2d0:op=0x55fdc4829fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 20 cr:s=0x55fdc4cfb2d0:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 20 cr:s=0x55fdc4cb4790:op=0x55fdc4831d50:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 20 cr:s=0x55fdc4cfb2d0:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 20 run: stack=0x55fdc4cfb2d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 20 cr:s=0x55fdc4cb4790:op=0x55fdc4cb4bc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 20 cr:s=0x55fdc4cb4790:op=0x55fdc4cb4bc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.479 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.480 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.480 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.480 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.480 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.480 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.480 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.480 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.480 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.480 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.480 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.480 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.480 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = e66b0f403555965582b72997bec9c4a05030123078e383080c69c43ee7c0823b 2026-03-20T11:47:14.480 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.480 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.480 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.480 INFO:teuthology.orchestra.run.vm02.stderr:e66b0f403555965582b72997bec9c4a05030123078e383080c69c43ee7c0823b 2026-03-20T11:47:14.480 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.480 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.480 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.480 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.480 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = b8568603b1a4f1f522716a40c046bb3a19010e0364bb55413763ca314f7cdc60 2026-03-20T11:47:14.480 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=b8568603b1a4f1f522716a40c046bb3a19010e0364bb55413763ca314f7cdc60 2026-03-20T11:47:14.480 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.480 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.480 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-14%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.480 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=31, curl_handle=0x55fdc4cc1a40 2026-03-20T11:47:14.480 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f48c3739900 20 run: stack=0x55fdc4cb4790 is io blocked 2026-03-20T11:47:14.480 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.479+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cc2150 req_data->id=31, curl_handle=0x55fdc4cc1a40 2026-03-20T11:47:14.509 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4d064e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.509 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4d064e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:14.509 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4d081c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.509 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f48c3739900 15 stack 0x55fdc4cf64d0 end 2026-03-20T11:47:14.509 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f48c3739900 20 run: stack=0x55fdc4cf64d0 is done 2026-03-20T11:47:14.509 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.509 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f48c3739900 20 collect(): s=0x55fdc4ca7e70 stack=0x55fdc4cf64d0 is complete 2026-03-20T11:47:14.509 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.509 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f48c3739900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:14.509 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f48c3739900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:14.509 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:14.509 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f48c3739900 15 stack 0x55fdc4ca7e70 end 2026-03-20T11:47:14.509 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f48c3739900 20 stack->operate() returned ret=-22 2026-03-20T11:47:14.509 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is done 2026-03-20T11:47:14.509 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.509 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f48c3739900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:14.509 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.509 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.509 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f48c3739900 4 starting trim on bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:47:14.509 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4829fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.509 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4829fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.509 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f48c3739900 20 enqueued request req=0x55fdc4cac740 2026-03-20T11:47:14.509 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f48c3739900 20 req: 0x55fdc4cac740 2026-03-20T11:47:14.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is io blocked 2026-03-20T11:47:14.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f4885ffb640 20 dequeued request req=0x55fdc4cac740 2026-03-20T11:47:14.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f4885ffb640 20 RGWWQ: empty 2026-03-20T11:47:14.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f4885ffb640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 : miss 2026-03-20T11:47:14.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.512+0000 7f4885ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.513+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.513+0000 7f4885ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.513+0000 7f4885ffb640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 info.flags=0x17 2026-03-20T11:47:14.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.513+0000 7f4885ffb640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 to cache LRU end 2026-03-20T11:47:14.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.513+0000 7f4885ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.513+0000 7f4885ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.513+0000 7f4885ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:47:14.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.513+0000 7f4885ffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-36 : miss 2026-03-20T11:47:14.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.513+0000 7f4885ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.513+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.513+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.513+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.513+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cab160:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.513+0000 7f48c3739900 15 stack 0x55fdc4cbbce0 end 2026-03-20T11:47:14.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.513+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is done 2026-03-20T11:47:14.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.513+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.513+0000 7f48c3739900 20 collect(): s=0x55fdc4bb3e20 stack=0x55fdc4cbbce0 is complete 2026-03-20T11:47:14.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.513+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.513+0000 7f48c3739900 10 trimming bilogs for bucket=:dipepu-19[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24]) markers=[,,,,,,,,00000000005.35.12,,], shards=11 2026-03-20T11:47:14.510 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.513+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4d081c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.513+0000 7f48c3739900 10 trimming bilog shard 8 of :dipepu-19[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24]) at marker 00000000005.35.12 2026-03-20T11:47:14.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.513+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.513+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cff1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.513+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cff1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.513+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24.8 2026-03-20T11:47:14.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.513+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is io blocked 2026-03-20T11:47:14.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f4885ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f4885ffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-36 info.flags=0x0 2026-03-20T11:47:14.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f4885ffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-36 to cache LRU end 2026-03-20T11:47:14.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f4885ffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-36 : miss 2026-03-20T11:47:14.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f4885ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f4885ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.511 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f4885ffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-36 info.flags=0x0 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f4885ffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-36 to cache LRU end 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f4885ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41): adding source pipe: {s={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f4885ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41): adding dest pipe: {s={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f4885ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f4885ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f4885ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4829fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4829fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4829fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4829fe0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cab160:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 2bbeb666889cf0f41c94be1d04c402f44d70ea8d41ccfaf3b4bb54a3e116e1ff 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.512 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2bbeb666889cf0f41c94be1d04c402f44d70ea8d41ccfaf3b4bb54a3e116e1ff 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = f5459dec83d0d7bbe5557aecd10a8de5917ac76c25735a233f733dae86cb3ec4 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=f5459dec83d0d7bbe5557aecd10a8de5917ac76c25735a233f733dae86cb3ec4 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-36%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=32, curl_handle=0x55fdc48e6af0 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f48c3739900 20 run: stack=0x55fdc4cf64d0 is io blocked 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.514+0000 7f481e7fc640 20 link_request req_data=0x55fdc48e75f0 req_data->id=32, curl_handle=0x55fdc48e6af0 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.515+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cff1f0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.515+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cff1f0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.515+0000 7f48c3739900 15 stack 0x55fdc4cbbce0 end 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.515+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.515+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is done 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.515+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4d081c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.515+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.515+0000 7f48c3739900 20 trim: trimmed bucket instance dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.515+0000 7f48c3739900 15 stack 0x55fdc4bb3e20 end 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.515+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is done 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.515+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.515+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.515+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.515+0000 7f48c3739900 4 starting trim on bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.515+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4830410:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.515+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4830410:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.515+0000 7f48c3739900 20 enqueued request req=0x55fdc4cc0410 2026-03-20T11:47:14.513 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.515+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.515+0000 7f48c3739900 20 req: 0x55fdc4cc0410 2026-03-20T11:47:14.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.515+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is io blocked 2026-03-20T11:47:14.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.515+0000 7f48857fa640 20 dequeued request req=0x55fdc4cc0410 2026-03-20T11:47:14.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.515+0000 7f48857fa640 20 RGWWQ: empty 2026-03-20T11:47:14.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.515+0000 7f48857fa640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 : miss 2026-03-20T11:47:14.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.515+0000 7f48857fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48857fa640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48857fa640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 info.flags=0x17 2026-03-20T11:47:14.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48857fa640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 to cache LRU end 2026-03-20T11:47:14.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48857fa640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48857fa640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48857fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:47:14.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48857fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-22 : miss 2026-03-20T11:47:14.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48857fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48857fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48857fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-22 info.flags=0x0 2026-03-20T11:47:14.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48857fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-22 to cache LRU end 2026-03-20T11:47:14.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48857fa640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-22 : miss 2026-03-20T11:47:14.514 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48857fa640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48857fa640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48857fa640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-22 info.flags=0x0 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48857fa640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-22 to cache LRU end 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48857fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27): adding source pipe: {s={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48857fa640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27): adding dest pipe: {s={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48857fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48857fa640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48857fa640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4830410:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4830410:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4830410:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4830410:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4d081c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4d064e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4d064e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.515 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.516 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 77095ff7c26ba12ed6c2049c7c1208396fc22fe60eac1e68dc63a67e46036ccf 2026-03-20T11:47:14.516 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.516 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.516 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.516 INFO:teuthology.orchestra.run.vm02.stderr:77095ff7c26ba12ed6c2049c7c1208396fc22fe60eac1e68dc63a67e46036ccf 2026-03-20T11:47:14.516 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.516 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.516 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.516 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.516 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = a08cb3b45740ea7b5e8714b0341585f5f73f768fb5b22a013aa10e0df9767582 2026-03-20T11:47:14.516 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a08cb3b45740ea7b5e8714b0341585f5f73f768fb5b22a013aa10e0df9767582 2026-03-20T11:47:14.516 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.516 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.516 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-22%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.516 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=33, curl_handle=0x55fdc4cbda00 2026-03-20T11:47:14.516 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.517+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is io blocked 2026-03-20T11:47:14.516 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.518+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cbe150 req_data->id=33, curl_handle=0x55fdc4cbda00 2026-03-20T11:47:14.516 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4d01100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.517 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4d01100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.517 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4d01100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.517 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4d01100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.517 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4cc6600:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.517 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 15 stack 0x55fdc4d025f0 end 2026-03-20T11:47:14.517 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 run: stack=0x55fdc4d025f0 is done 2026-03-20T11:47:14.517 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.517 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 collect(): s=0x55fdc4bc6f60 stack=0x55fdc4d025f0 is complete 2026-03-20T11:47:14.517 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.517 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 10 trimming bilogs for bucket=:dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) markers=[00000000004.87.6,00000000002.126.1,00000000004.75.6,00000000002.34.1,00000000004.89.6,00000000002.87.1,00000000002.128.1,00000000004.94.6,00000000002.37.1,00000000002.73.1,00000000002.61.1], shards=11 2026-03-20T11:47:14.517 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cc6600:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.517 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 10 trimming bilog shard 0 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.87.6 2026-03-20T11:47:14.517 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 10 trimming bilog shard 1 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.126.1 2026-03-20T11:47:14.517 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 10 trimming bilog shard 2 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.75.6 2026-03-20T11:47:14.517 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 10 trimming bilog shard 3 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.34.1 2026-03-20T11:47:14.517 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 10 trimming bilog shard 4 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.89.6 2026-03-20T11:47:14.517 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 10 trimming bilog shard 5 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.87.1 2026-03-20T11:47:14.517 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 10 trimming bilog shard 6 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.128.1 2026-03-20T11:47:14.517 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 10 trimming bilog shard 7 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000004.94.6 2026-03-20T11:47:14.517 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 10 trimming bilog shard 8 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.37.1 2026-03-20T11:47:14.517 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 10 trimming bilog shard 9 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.73.1 2026-03-20T11:47:14.517 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 10 trimming bilog shard 10 of :dipepu-53[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13]) at marker 00000000002.61.1 2026-03-20T11:47:14.517 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.517 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4cc48a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.517 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc482d400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4cb1a00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cb0d70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4830410:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cb5d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc4cb48d0:op=0x55fdc4d1c830:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc48e6940:op=0x55fdc4d1e200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc4cf5bf0:op=0x55fdc4d1ea00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4d1f200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4d1fa00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4cc48a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.0 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 run: stack=0x55fdc4d025f0 is io blocked 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc482d400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.1 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 run: stack=0x55fdc4d01f90 is io blocked 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4cb1a00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.2 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is io blocked 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cb0d70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.3 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 run: stack=0x55fdc4ca7520 is io blocked 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4830410:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.4 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 run: stack=0x55fdc4ca7c70 is io blocked 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cb5d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.5 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is io blocked 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc4cb48d0:op=0x55fdc4d1c830:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.6 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 run: stack=0x55fdc4cb48d0 is io blocked 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc48e6940:op=0x55fdc4d1e200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.7 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 run: stack=0x55fdc48e6940 is io blocked 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc4cf5bf0:op=0x55fdc4d1ea00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.8 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 run: stack=0x55fdc4cf5bf0 is io blocked 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4d1f200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.9 2026-03-20T11:47:14.518 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 run: stack=0x55fdc4ca6700 is io blocked 2026-03-20T11:47:14.519 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4d1fa00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.519 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13.10 2026-03-20T11:47:14.519 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.520+0000 7f48c3739900 20 run: stack=0x55fdc4cf5990 is io blocked 2026-03-20T11:47:14.519 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.521+0000 7f48c3739900 20 cr:s=0x55fdc4cb4790:op=0x55fdc4cb4bc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.520 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4cb4790:op=0x55fdc4cb4bc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.520 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4830410:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.520 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4830410:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.520 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 15 stack 0x55fdc4ca7c70 end 2026-03-20T11:47:14.520 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.520 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 run: stack=0x55fdc4ca7c70 is done 2026-03-20T11:47:14.520 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4d1fa00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.520 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4d1fa00:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.520 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 15 stack 0x55fdc4cf5990 end 2026-03-20T11:47:14.520 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.520 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 run: stack=0x55fdc4cf5990 is done 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4cb1a00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4cb1a00:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 15 stack 0x55fdc48e4c10 end 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is done 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4d1f200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4d1f200:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 15 stack 0x55fdc4ca6700 end 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 run: stack=0x55fdc4ca6700 is done 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cb0d70:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc4cb0d70:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 15 stack 0x55fdc4ca7520 end 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 run: stack=0x55fdc4ca7520 is done 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cb5d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cb5d20:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 15 stack 0x55fdc4cf32a0 end 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is done 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4cc48a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4cc48a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 15 stack 0x55fdc4d025f0 end 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 run: stack=0x55fdc4d025f0 is done 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4cb4790:op=0x55fdc4cb4bc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cc6600:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4cf5bf0:op=0x55fdc4d1ea00:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4cf5bf0:op=0x55fdc4d1ea00:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 15 stack 0x55fdc4cf5bf0 end 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 run: stack=0x55fdc4cf5bf0 is done 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc48e6940:op=0x55fdc4d1e200:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc48e6940:op=0x55fdc4d1e200:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 15 stack 0x55fdc48e6940 end 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 run: stack=0x55fdc48e6940 is done 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc482d400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc482d400:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 15 stack 0x55fdc4d01f90 end 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 run: stack=0x55fdc4d01f90 is done 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4cb4790:op=0x55fdc4cb4bc0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cc6600:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4cb4790:op=0x55fdc4831d50:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 15 stack 0x55fdc4cb4790 end 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 run: stack=0x55fdc4cb4790 is done 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cc6600:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4cfb2d0:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 collect(): s=0x55fdc4cfb2d0 stack=0x55fdc4cb4790 is complete 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 10 trimming bilogs for bucket=:dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) markers=[,,,,,00000000003.36.10,,,00000000011.47.6,00000000003.43.6,], shards=11 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cc6600:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.521 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4cfb2d0:op=0x55fdc4831d50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.522 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 10 trimming bilog shard 5 of :dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) at marker 00000000003.36.10 2026-03-20T11:47:14.522 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 10 trimming bilog shard 8 of :dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) at marker 00000000011.47.6 2026-03-20T11:47:14.522 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 10 trimming bilog shard 9 of :dipepu-14[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19]) at marker 00000000003.43.6 2026-03-20T11:47:14.522 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 run: stack=0x55fdc4cfb2d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.522 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cc6600:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.522 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc482d400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.522 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4cb4790:op=0x55fdc4cb5d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.522 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4830410:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.522 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cc6600:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.522 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc482d400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.522 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19.5 2026-03-20T11:47:14.522 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 run: stack=0x55fdc4ca7520 is io blocked 2026-03-20T11:47:14.522 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4cb4790:op=0x55fdc4cb5d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.522 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19.8 2026-03-20T11:47:14.522 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 run: stack=0x55fdc4cb4790 is io blocked 2026-03-20T11:47:14.522 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4830410:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.522 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19.9 2026-03-20T11:47:14.522 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is io blocked 2026-03-20T11:47:14.522 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cc6600:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.522 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cc6600:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.522 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cc6600:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.522 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cc6600:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.522 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.523+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.524+0000 7f48c3739900 20 cr:s=0x55fdc4cb48d0:op=0x55fdc4d1c830:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.524+0000 7f48c3739900 20 cr:s=0x55fdc4cb48d0:op=0x55fdc4d1c830:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.524+0000 7f48c3739900 15 stack 0x55fdc4cb48d0 end 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.524+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.524+0000 7f48c3739900 20 run: stack=0x55fdc4cb48d0 is done 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.524+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cc6600:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.524+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.524+0000 7f48c3739900 20 trim: trimmed bucket instance dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.524+0000 7f48c3739900 15 stack 0x55fdc4bc6f60 end 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.524+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is done 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.524+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.524+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.524+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.524+0000 7f48c3739900 4 starting trim on bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.524+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cff1f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.524+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cff1f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.524+0000 7f48c3739900 20 enqueued request req=0x55fdc4cc38d0 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.524+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.524+0000 7f48c3739900 20 req: 0x55fdc4cc38d0 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.524+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is io blocked 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.524+0000 7f4884ff9640 20 dequeued request req=0x55fdc4cc38d0 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.524+0000 7f4884ff9640 20 RGWWQ: empty 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.524+0000 7f4884ff9640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 : miss 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.524+0000 7f4884ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4830410:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4830410:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 15 stack 0x55fdc48e4c10 end 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is done 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 cr:s=0x55fdc4cfb2d0:op=0x55fdc4831d50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 run: stack=0x55fdc4cfb2d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 cr:s=0x55fdc4cb4790:op=0x55fdc4cb5d20:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 cr:s=0x55fdc4cb4790:op=0x55fdc4cb5d20:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 15 stack 0x55fdc4cb4790 end 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.523 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 run: stack=0x55fdc4cb4790 is done 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 cr:s=0x55fdc4cfb2d0:op=0x55fdc4831d50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 run: stack=0x55fdc4cfb2d0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc482d400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f4884ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 cr:s=0x55fdc4ca7520:op=0x55fdc482d400:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 15 stack 0x55fdc4ca7520 end 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 run: stack=0x55fdc4ca7520 is done 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 cr:s=0x55fdc4cfb2d0:op=0x55fdc4831d50:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 cr:s=0x55fdc4cfb2d0:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 trim: trimmed bucket instance dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 15 stack 0x55fdc4cfb2d0 end 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f4884ff9640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 info.flags=0x17 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 run: stack=0x55fdc4cfb2d0 is done 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f4884ff9640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 to cache LRU end 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f4884ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f4884ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 4 starting trim on bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4cb3150:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4cb3150:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 enqueued request req=0x55fdc4cae180 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 req: 0x55fdc4cae180 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f4884ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is io blocked 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f4884ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-25 : miss 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f4884ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f487bfff640 20 dequeued request req=0x55fdc4cae180 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f487bfff640 20 RGWWQ: empty 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f487bfff640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 : miss 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f487bfff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f4884ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f4884ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-25 info.flags=0x0 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f4884ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-25 to cache LRU end 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f4884ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-25 : miss 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f4884ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f487bfff640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f487bfff640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 info.flags=0x17 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f487bfff640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 to cache LRU end 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f487bfff640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f487bfff640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f487bfff640 10 rgw async rados processor: updating xattr: name=user.rgw.cors bl.length()=69 2026-03-20T11:47:14.524 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f487bfff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:47:14.525 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f487bfff640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-35 : miss 2026-03-20T11:47:14.525 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.525+0000 7f487bfff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.525 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f4884ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.525 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f4884ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-25 info.flags=0x0 2026-03-20T11:47:14.525 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f4884ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-25 to cache LRU end 2026-03-20T11:47:14.525 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f4884ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30): adding source pipe: {s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.525 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f4884ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30): adding dest pipe: {s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.525 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f4884ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.525 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f4884ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.525 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f4884ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30 2026-03-20T11:47:14.525 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cff1f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.525 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cff1f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cff1f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cff1f0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f487bfff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4831d50:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4d01100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4d01100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f487bfff640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-35 info.flags=0x0 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f487bfff640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-35 to cache LRU end 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f487bfff640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-35 : miss 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f487bfff640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 385939dc0f75d52d43b5245c4b7d2cc2d53dc4a422d3938173d3d16310b154c4 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:385939dc0f75d52d43b5245c4b7d2cc2d53dc4a422d3938173d3d16310b154c4 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 922fa38f02ba45fa1e9aff69280ed3133904cb862199037baa04e215c25eb64e 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=922fa38f02ba45fa1e9aff69280ed3133904cb862199037baa04e215c25eb64e 2026-03-20T11:47:14.526 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-25%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=34, curl_handle=0x55fdc4cc1a40 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f48c3739900 20 run: stack=0x55fdc4cf5990 is io blocked 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cc2150 req_data->id=34, curl_handle=0x55fdc4cc1a40 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f487bfff640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f487bfff640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-35 info.flags=0x0 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f487bfff640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-35 to cache LRU end 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f487bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40): adding source pipe: {s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f487bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40): adding dest pipe: {s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f487bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f487bfff640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.526+0000 7f487bfff640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4cb3150:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4cb3150:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4cb3150:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4cb3150:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc482bc70:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4cc6600:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4cc6600:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 39253cac4bcb17fba50221d91445a1de36bcb0a0b8230cdfe23f774f259b5705 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:39253cac4bcb17fba50221d91445a1de36bcb0a0b8230cdfe23f774f259b5705 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 54574b97a64f92facd639655d458b4f43b99f8c450c9bb858e7fe00475f69c96 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=54574b97a64f92facd639655d458b4f43b99f8c450c9bb858e7fe00475f69c96 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-35%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=35, curl_handle=0x55fdc4cb95e0 2026-03-20T11:47:14.527 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f48c3739900 20 run: stack=0x55fdc4ca6700 is io blocked 2026-03-20T11:47:14.528 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.527+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cb9d30 req_data->id=35, curl_handle=0x55fdc4cb95e0 2026-03-20T11:47:14.528 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:47:14.526+0000 7f12701d0640 -1 req 16922764578893790081 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:47:14.528 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:47:14.527+0000 7f12a3236640 -1 req 14963557586120466158 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:47:14.554 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.557+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.554 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.557+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.554 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.557+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.554 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.557+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.554 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.557+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cab160:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.554 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.557+0000 7f48c3739900 15 stack 0x55fdc4cf64d0 end 2026-03-20T11:47:14.554 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.557+0000 7f48c3739900 20 run: stack=0x55fdc4cf64d0 is done 2026-03-20T11:47:14.554 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.557+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.554 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.557+0000 7f48c3739900 20 collect(): s=0x55fdc4ca7e70 stack=0x55fdc4cf64d0 is complete 2026-03-20T11:47:14.554 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.557+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.554 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.557+0000 7f48c3739900 10 trimming bilogs for bucket=:dipepu-36[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41]) markers=[,,,,,,,,,,00000000001.53.6], shards=11 2026-03-20T11:47:14.554 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.557+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cab160:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.554 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.557+0000 7f48c3739900 10 trimming bilog shard 10 of :dipepu-36[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41]) at marker 00000000001.53.6 2026-03-20T11:47:14.554 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.557+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.554 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.557+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc482d400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.554 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.557+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc482d400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.554 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.557+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41.10 2026-03-20T11:47:14.554 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.557+0000 7f48c3739900 20 run: stack=0x55fdc4cf64d0 is io blocked 2026-03-20T11:47:14.555 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.558+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc482d400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.555 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.558+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc482d400:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.555 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.558+0000 7f48c3739900 15 stack 0x55fdc4cf64d0 end 2026-03-20T11:47:14.555 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.558+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.555 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.558+0000 7f48c3739900 20 run: stack=0x55fdc4cf64d0 is done 2026-03-20T11:47:14.555 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.558+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cab160:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.555 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.558+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.555 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.558+0000 7f48c3739900 20 trim: trimmed bucket instance dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.558+0000 7f48c3739900 15 stack 0x55fdc4ca7e70 end 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.558+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is done 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.558+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.558+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.558+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.558+0000 7f48c3739900 4 starting trim on bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.558+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cf6b90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.558+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cf6b90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.558+0000 7f48c3739900 20 enqueued request req=0x55fdc4cac740 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.558+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.558+0000 7f48c3739900 20 req: 0x55fdc4cac740 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.558+0000 7f487b7fe640 20 dequeued request req=0x55fdc4cac740 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.558+0000 7f487b7fe640 20 RGWWQ: empty 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.558+0000 7f487b7fe640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 : miss 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.558+0000 7f487b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.558+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is io blocked 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.559+0000 7f487b7fe640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.559+0000 7f487b7fe640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 info.flags=0x17 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.559+0000 7f487b7fe640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 to cache LRU end 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.559+0000 7f487b7fe640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.559+0000 7f487b7fe640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.559+0000 7f487b7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.559+0000 7f487b7fe640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-26 : miss 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.559+0000 7f487b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.559+0000 7f487b7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.559+0000 7f487b7fe640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-26 info.flags=0x0 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.559+0000 7f487b7fe640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-26 to cache LRU end 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.559+0000 7f487b7fe640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-26 : miss 2026-03-20T11:47:14.556 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.559+0000 7f487b7fe640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.559+0000 7f487b7fe640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.559+0000 7f487b7fe640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-26 info.flags=0x0 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.559+0000 7f487b7fe640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-26 to cache LRU end 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.559+0000 7f487b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31): adding source pipe: {s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.559+0000 7f487b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31): adding dest pipe: {s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.559+0000 7f487b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.559+0000 7f487b7fe640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.559+0000 7f487b7fe640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cf6b90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cf6b90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cf6b90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cf6b90:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cab160:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = b052901259c8d71062e5858cb25d372a54b3e645861313b0408ec68e5f8268f7 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:b052901259c8d71062e5858cb25d372a54b3e645861313b0408ec68e5f8268f7 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 93970d89379fe665cc8029544dd266aa43a280025e4f742ee2894be3098545f7 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=93970d89379fe665cc8029544dd266aa43a280025e4f742ee2894be3098545f7 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-26%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.557 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=36, curl_handle=0x55fdc4cbda00 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 run: stack=0x55fdc4cf64d0 is io blocked 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4d064e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f481e7fc640 20 link_request req_data=0x55fdc48e75f0 req_data->id=36, curl_handle=0x55fdc4cbda00 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4d064e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4d064e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4d064e0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4d081c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 15 stack 0x55fdc4cbbce0 end 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is done 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 collect(): s=0x55fdc4bb3e20 stack=0x55fdc4cbbce0 is complete 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 10 trimming bilogs for bucket=:dipepu-22[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27]) markers=[,,,,,,,,00000000005.29.12,,], shards=11 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cf6b90:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 10 trimming bilog shard 8 of :dipepu-22[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27]) at marker 00000000005.29.12 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc482d400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc482d400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27.8 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.560+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is io blocked 2026-03-20T11:47:14.558 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:47:14.560+0000 7f12b525a640 -1 req 16107652939564353116 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc482d400:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc482d400:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f48c3739900 15 stack 0x55fdc4cbbce0 end 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is done 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cf6b90:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f48c3739900 20 trim: trimmed bucket instance dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f48c3739900 15 stack 0x55fdc4bb3e20 end 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is done 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.558 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f48c3739900 4 starting trim on bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc482d400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc482d400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f48c3739900 20 enqueued request req=0x55fdc4cc0410 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f48c3739900 20 req: 0x55fdc4cc0410 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f487affd640 20 dequeued request req=0x55fdc4cc0410 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f487affd640 20 RGWWQ: empty 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f487affd640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 : miss 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f487affd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is io blocked 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f487affd640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f487affd640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 info.flags=0x17 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f487affd640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 to cache LRU end 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f487affd640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f487affd640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f487affd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f487affd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-51 : miss 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.561+0000 7f487affd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f487affd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f487affd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-51 info.flags=0x0 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f487affd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-51 to cache LRU end 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f487affd640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-51 : miss 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f487affd640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f487affd640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f487affd640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-51 info.flags=0x0 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f487affd640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-51 to cache LRU end 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f487affd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11): adding source pipe: {s={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f487affd640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11): adding dest pipe: {s={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f487affd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f487affd640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f487affd640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc482d400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc482d400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc482d400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc482d400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4d081c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.559 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4831640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4831640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 134c7a600c14b618093da202575c14e7864198d5e478d5bcdfef1d7885fe465d 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:134c7a600c14b618093da202575c14e7864198d5e478d5bcdfef1d7885fe465d 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = a48de6bdcf9e152f1507106fc81029077ba32ea6615b1a6babdf673afbdd86cf 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=a48de6bdcf9e152f1507106fc81029077ba32ea6615b1a6babdf673afbdd86cf 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-51%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=37, curl_handle=0x55fdc48e6af0 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is io blocked 2026-03-20T11:47:14.560 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.562+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cbe150 req_data->id=37, curl_handle=0x55fdc48e6af0 2026-03-20T11:47:14.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4d01100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4d01100:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:14.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4831d50:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 15 stack 0x55fdc4cf5990 end 2026-03-20T11:47:14.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 run: stack=0x55fdc4cf5990 is done 2026-03-20T11:47:14.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4cc6600:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4cc6600:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:14.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 collect(): s=0x55fdc4bc6f60 stack=0x55fdc4cf5990 is complete 2026-03-20T11:47:14.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:14.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:14.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:14.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 15 stack 0x55fdc4bc6f60 end 2026-03-20T11:47:14.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 stack->operate() returned ret=-22 2026-03-20T11:47:14.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is done 2026-03-20T11:47:14.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc482bc70:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 15 stack 0x55fdc4ca6700 end 2026-03-20T11:47:14.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 run: stack=0x55fdc4ca6700 is done 2026-03-20T11:47:14.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:14.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.565 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 collect(): s=0x55fdc48e4c10 stack=0x55fdc4ca6700 is complete 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 15 stack 0x55fdc48e4c10 end 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 stack->operate() returned ret=-22 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is done 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 4 starting trim on bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cfaee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 4 starting trim on bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cfaee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 enqueued request req=0x55fdc4d05710 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 req: 0x55fdc4d05710 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f487a7fc640 20 dequeued request req=0x55fdc4d05710 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f487a7fc640 20 RGWWQ: empty 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f487a7fc640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 : miss 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f487a7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is io blocked 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc482d400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc482d400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 enqueued request req=0x55fdc4d08990 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 req: 0x55fdc4d08990 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f4879ffb640 20 dequeued request req=0x55fdc4d08990 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f4879ffb640 20 RGWWQ: empty 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f4879ffb640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 : miss 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f4879ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.566 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is io blocked 2026-03-20T11:47:14.568 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f487a7fc640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.568 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f487a7fc640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 info.flags=0x17 2026-03-20T11:47:14.568 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f487a7fc640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 to cache LRU end 2026-03-20T11:47:14.568 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f487a7fc640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.568 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f487a7fc640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.568 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f487a7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:47:14.568 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f487a7fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-55 : miss 2026-03-20T11:47:14.568 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f487a7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.568 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f4879ffb640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.568 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f4879ffb640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 info.flags=0x17 2026-03-20T11:47:14.568 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f4879ffb640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 to cache LRU end 2026-03-20T11:47:14.568 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f4879ffb640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.568 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f4879ffb640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:47:14.568 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f4879ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:47:14.568 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f4879ffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-28 : miss 2026-03-20T11:47:14.568 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.568+0000 7f4879ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.568 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f487a7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.568 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f487a7fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-55 info.flags=0x0 2026-03-20T11:47:14.568 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f487a7fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-55 to cache LRU end 2026-03-20T11:47:14.568 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f487a7fc640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-55 : miss 2026-03-20T11:47:14.568 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f487a7fc640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.568 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f4879ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.568 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f4879ffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-28 info.flags=0x0 2026-03-20T11:47:14.568 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f4879ffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-28 to cache LRU end 2026-03-20T11:47:14.568 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f4879ffb640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-28 : miss 2026-03-20T11:47:14.568 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f4879ffb640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f4879ffb640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f4879ffb640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-28 info.flags=0x0 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f4879ffb640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-28 to cache LRU end 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f4879ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding source pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f4879ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33): adding dest pipe: {s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f4879ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f4879ffb640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f4879ffb640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f487a7fc640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f487a7fc640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-55 info.flags=0x0 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f487a7fc640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-55 to cache LRU end 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f487a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding source pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f487a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15): adding dest pipe: {s={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f487a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f487a7fc640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f487a7fc640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc482d400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc482d400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cfaee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc482d400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cfaee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc482d400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cfaee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cfaee0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4cc6600:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4d01100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4831d30:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4d01100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.569 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 4977f8c81e55d4e44ed4726bb5f6f6c16f816ddb281a74cd71a070b09e02c3d9 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:4977f8c81e55d4e44ed4726bb5f6f6c16f816ddb281a74cd71a070b09e02c3d9 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 8066a2c85ffefa5f24dd6d5993240e8302e4d0c92e1729813d46cea5cd8e118b 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=8066a2c85ffefa5f24dd6d5993240e8302e4d0c92e1729813d46cea5cd8e118b 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-28%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=38, curl_handle=0x55fdc4cb95e0 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 run: stack=0x55fdc4ca6700 is io blocked 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4cfaee0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4cfaee0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.570 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cb9d30 req_data->id=38, curl_handle=0x55fdc4cb95e0 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 35408b985f4e227904cc637ce6efa6384e35c6213c67a699ad6fef2fd052e3b7 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:35408b985f4e227904cc637ce6efa6384e35c6213c67a699ad6fef2fd052e3b7 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = c372af20e13262442d5f35d67de5bca7d4d05e9c64e1e361cd7235642d3b7270 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=c372af20e13262442d5f35d67de5bca7d4d05e9c64e1e361cd7235642d3b7270 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-55%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=39, curl_handle=0x55fdc4cc1a40 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f48c3739900 20 run: stack=0x55fdc4cf5990 is io blocked 2026-03-20T11:47:14.571 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.569+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cc2150 req_data->id=39, curl_handle=0x55fdc4cc1a40 2026-03-20T11:47:14.572 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:47:14.570+0000 7f12ac248640 -1 req 11641303184748443088 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:47:14.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:14.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cab160:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f48c3739900 15 stack 0x55fdc4cf64d0 end 2026-03-20T11:47:14.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f48c3739900 20 run: stack=0x55fdc4cf64d0 is done 2026-03-20T11:47:14.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f48c3739900 20 collect(): s=0x55fdc4ca7e70 stack=0x55fdc4cf64d0 is complete 2026-03-20T11:47:14.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f48c3739900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:14.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f48c3739900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:14.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:14.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f48c3739900 15 stack 0x55fdc4ca7e70 end 2026-03-20T11:47:14.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f48c3739900 20 stack->operate() returned ret=-22 2026-03-20T11:47:14.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is done 2026-03-20T11:47:14.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f48c3739900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:14.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f48c3739900 4 starting trim on bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:47:14.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc482d400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc482d400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f48c3739900 20 enqueued request req=0x55fdc4cac740 2026-03-20T11:47:14.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f48c3739900 20 req: 0x55fdc4cac740 2026-03-20T11:47:14.598 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is io blocked 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f4878ff9640 20 dequeued request req=0x55fdc4cac740 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f4878ff9640 20 RGWWQ: empty 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f4878ff9640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 : miss 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f4878ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f4878ff9640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f4878ff9640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 info.flags=0x17 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f4878ff9640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 to cache LRU end 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f4878ff9640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f4878ff9640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=247 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f4878ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f4878ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-31 : miss 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.601+0000 7f4878ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f4878ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f4878ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-31 info.flags=0x0 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f4878ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-31 to cache LRU end 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f4878ff9640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-31 : miss 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f4878ff9640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f4878ff9640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f4878ff9640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-31 info.flags=0x0 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f4878ff9640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-31 to cache LRU end 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f4878ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding source pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f4878ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36): adding dest pipe: {s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f4878ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f4878ff9640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f4878ff9640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc482d400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc482d400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc482d400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc482d400:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.599 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cab160:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = baa79ca103513487fd17f83769596c3ef6c01809a6f958ddfa6a7df3cb8e0625 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:baa79ca103513487fd17f83769596c3ef6c01809a6f958ddfa6a7df3cb8e0625 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 9421484fe219b39a50667d4cf1f9920f3991dc95c5ef26becbdb28b35d98fcbf 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=9421484fe219b39a50667d4cf1f9920f3991dc95c5ef26becbdb28b35d98fcbf 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-31%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=40, curl_handle=0x55fdc4cbda00 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f48c3739900 20 run: stack=0x55fdc4cf64d0 is io blocked 2026-03-20T11:47:14.600 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.602+0000 7f481e7fc640 20 link_request req_data=0x55fdc48e75f0 req_data->id=40, curl_handle=0x55fdc4cbda00 2026-03-20T11:47:14.600 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:47:14.603+0000 7f128c208640 -1 req 2756982573914800312 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:47:14.601 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4831640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4831640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4831640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4831640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4d081c0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 15 stack 0x55fdc4cbbce0 end 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is done 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 20 collect(): s=0x55fdc4bb3e20 stack=0x55fdc4cbbce0 is complete 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 10 trimming bilogs for bucket=:dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) markers=[00000000002.62.6,,00000000002.48.6,,00000000002.70.6,,,00000000002.59.6,,,], shards=11 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cf5240:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 10 trimming bilog shard 0 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.62.6 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 10 trimming bilog shard 2 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.48.6 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 10 trimming bilog shard 4 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.70.6 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 10 trimming bilog shard 7 of :dipepu-51[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11]) at marker 00000000002.59.6 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cb6af0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc482b530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cb52e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cc48a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cb6af0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.0 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is io blocked 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc482b530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.2 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is io blocked 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cb52e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.4 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 20 run: stack=0x55fdc4ca7c70 is io blocked 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cc48a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11.7 2026-03-20T11:47:14.602 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.605+0000 7f48c3739900 20 run: stack=0x55fdc4d01f90 is io blocked 2026-03-20T11:47:14.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.606+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc482b530:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.606+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc482b530:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.606+0000 7f48c3739900 15 stack 0x55fdc4cf32a0 end 2026-03-20T11:47:14.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.606+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.606+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is done 2026-03-20T11:47:14.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.606+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cf5240:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.603 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.606+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cb6af0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cb6af0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 15 stack 0x55fdc4cbbce0 end 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is done 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cf5240:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cc48a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cc48a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 15 stack 0x55fdc4d01f90 end 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 20 run: stack=0x55fdc4d01f90 is done 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cb52e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cb52e0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 15 stack 0x55fdc4ca7c70 end 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 20 run: stack=0x55fdc4ca7c70 is done 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cf5240:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cf5240:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 20 trim: trimmed bucket instance dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 15 stack 0x55fdc4bb3e20 end 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is done 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 4 starting trim on bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4829fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4829fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 20 enqueued request req=0x55fdc4cf6880 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 20 req: 0x55fdc4cf6880 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is io blocked 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48787f8640 20 dequeued request req=0x55fdc4cf6880 2026-03-20T11:47:14.604 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48787f8640 20 RGWWQ: empty 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48787f8640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 : miss 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48787f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48787f8640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48787f8640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 info.flags=0x17 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48787f8640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 to cache LRU end 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48787f8640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48787f8640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48787f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48787f8640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-39 : miss 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.607+0000 7f48787f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48787f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48787f8640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-39 info.flags=0x0 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48787f8640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-39 to cache LRU end 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48787f8640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-39 : miss 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48787f8640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48787f8640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48787f8640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-39 info.flags=0x0 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48787f8640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-39 to cache LRU end 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48787f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding source pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48787f8640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44): adding dest pipe: {s={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48787f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48787f8640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48787f8640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4829fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4829fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4829fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4829fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc482c640:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.605 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4831640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4831640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 39a7afdd231d034d90906b214c9c34e13a4a0a791449244663f996cf9dcf6fd8 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:39a7afdd231d034d90906b214c9c34e13a4a0a791449244663f996cf9dcf6fd8 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = d23776554479c266536a11c7692d8eedf090677b688a7078691bfbc2eb4b408a 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=d23776554479c266536a11c7692d8eedf090677b688a7078691bfbc2eb4b408a 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-39%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=41, curl_handle=0x55fdc48e6af0 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f48c3739900 20 run: stack=0x55fdc4d01f90 is io blocked 2026-03-20T11:47:14.606 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.608+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cbe150 req_data->id=41, curl_handle=0x55fdc48e6af0 2026-03-20T11:47:14.607 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4d01100:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4d01100:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4cc6600:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f48c3739900 15 stack 0x55fdc4ca6700 end 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f48c3739900 20 run: stack=0x55fdc4ca6700 is done 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f48c3739900 20 collect(): s=0x55fdc48e4c10 stack=0x55fdc4ca6700 is complete 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f48c3739900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f48c3739900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f48c3739900 15 stack 0x55fdc48e4c10 end 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f48c3739900 20 stack->operate() returned ret=-22 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is done 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f48c3739900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f48c3739900 4 starting trim on bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4829fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4829fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f48c3739900 20 enqueued request req=0x55fdc4cfeb50 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f48c3739900 20 req: 0x55fdc4cfeb50 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is io blocked 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f4877ff7640 20 dequeued request req=0x55fdc4cfeb50 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f4877ff7640 20 RGWWQ: empty 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f4877ff7640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 : miss 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f4877ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f4877ff7640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f4877ff7640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 info.flags=0x17 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f4877ff7640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 to cache LRU end 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f4877ff7640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f4877ff7640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f4877ff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f4877ff7640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-8 : miss 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f4877ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f4877ff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f4877ff7640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-8 info.flags=0x0 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f4877ff7640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-8 to cache LRU end 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f4877ff7640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-8 : miss 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.611+0000 7f4877ff7640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f4877ff7640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f4877ff7640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-8 info.flags=0x0 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f4877ff7640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-8 to cache LRU end 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f4877ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding source pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4cfaee0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f4877ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13): adding dest pipe: {s={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.608 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f4877ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f4877ff7640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f4877ff7640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4cfaee0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4829fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4cfaee0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4829fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4cfaee0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4829fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4831d30:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 15 stack 0x55fdc4cf5990 end 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 run: stack=0x55fdc4cf5990 is done 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4829fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 collect(): s=0x55fdc4bc6f60 stack=0x55fdc4cf5990 is complete 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 10 trimming bilogs for bucket=:dipepu-55[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15]) markers=[,,,,,00000000001.133.6,,,,,], shards=11 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc482ddf0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 10 trimming bilog shard 5 of :dipepu-55[4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15]) at marker 00000000001.133.6 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4cfaee0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4cf15a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4cb1f80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4cf15a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15.5 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 run: stack=0x55fdc4ca6700 is io blocked 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4cb1f80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.609 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = bc6d3320bd8a0be4849122ed9d0c2e9c822a59fbb488e26ac68fb4e5909bee5b 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:bc6d3320bd8a0be4849122ed9d0c2e9c822a59fbb488e26ac68fb4e5909bee5b 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 3113e001c83fdbe793141fdf0b2c57ad3bb3d99dbbd0600f2ec77bbcd8f3fdf8 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=3113e001c83fdbe793141fdf0b2c57ad3bb3d99dbbd0600f2ec77bbcd8f3fdf8 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-8%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=42, curl_handle=0x55fdc4cc1a40 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f48c3739900 20 run: stack=0x55fdc4cf5990 is io blocked 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.612+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cc2150 req_data->id=42, curl_handle=0x55fdc4cc1a40 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4cf15a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4cf15a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.610 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 15 stack 0x55fdc4ca6700 end 2026-03-20T11:47:14.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 20 run: stack=0x55fdc4ca6700 is done 2026-03-20T11:47:14.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc482ddf0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 20 trim: trimmed bucket instance dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15 2026-03-20T11:47:14.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 15 stack 0x55fdc4bc6f60 end 2026-03-20T11:47:14.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is done 2026-03-20T11:47:14.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 4 starting trim on bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:47:14.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4829fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4829fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 20 enqueued request req=0x55fdc4d08990 2026-03-20T11:47:14.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 20 req: 0x55fdc4d08990 2026-03-20T11:47:14.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is io blocked 2026-03-20T11:47:14.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48777f6640 20 dequeued request req=0x55fdc4d08990 2026-03-20T11:47:14.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48777f6640 20 RGWWQ: empty 2026-03-20T11:47:14.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48777f6640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 : miss 2026-03-20T11:47:14.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48777f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48777f6640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48777f6640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 info.flags=0x17 2026-03-20T11:47:14.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48777f6640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 to cache LRU end 2026-03-20T11:47:14.611 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48777f6640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48777f6640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48777f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48777f6640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-16 : miss 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48777f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48777f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48777f6640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-16 info.flags=0x0 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48777f6640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-16 to cache LRU end 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48777f6640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-16 : miss 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48777f6640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48777f6640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48777f6640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-16 info.flags=0x0 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48777f6640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-16 to cache LRU end 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48777f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding source pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48777f6640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21): adding dest pipe: {s={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48777f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48777f6640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48777f6640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4829fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4829fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4829fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4829fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4831d30:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4cf6b90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4cf6b90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.614+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.615+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.615+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.615+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.615+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.615+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.615+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = e4ed0dd7de16689fe7faa9e54aee2e7b60d86886136d8b50f66cdbb7483d9df3 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.615+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:e4ed0dd7de16689fe7faa9e54aee2e7b60d86886136d8b50f66cdbb7483d9df3 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.615+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.615+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.615+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.615+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.615+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 8552d39a0cb8789589958a8337f179d6e608806b4d525b04e4f7e8ba5cd1d24e 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.615+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=8552d39a0cb8789589958a8337f179d6e608806b4d525b04e4f7e8ba5cd1d24e 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.615+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.612 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.615+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.613 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.615+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-16%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.613 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.615+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=43, curl_handle=0x55fdc4cb95e0 2026-03-20T11:47:14.613 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.615+0000 7f48c3739900 20 run: stack=0x55fdc4ca6700 is io blocked 2026-03-20T11:47:14.613 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.615+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cb9d30 req_data->id=43, curl_handle=0x55fdc4cb95e0 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cab160:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f48c3739900 15 stack 0x55fdc4cf64d0 end 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f48c3739900 20 run: stack=0x55fdc4cf64d0 is done 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f48c3739900 20 collect(): s=0x55fdc4ca7e70 stack=0x55fdc4cf64d0 is complete 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f48c3739900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f48c3739900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f48c3739900 15 stack 0x55fdc4ca7e70 end 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f48c3739900 20 stack->operate() returned ret=-22 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is done 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f48c3739900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f48c3739900 4 starting trim on bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4829fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4829fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f48c3739900 20 enqueued request req=0x55fdc4cac740 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f48c3739900 20 req: 0x55fdc4cac740 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is io blocked 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f4876ff5640 20 dequeued request req=0x55fdc4cac740 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f4876ff5640 20 RGWWQ: empty 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f4876ff5640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 : miss 2026-03-20T11:47:14.641 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f4876ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f4876ff5640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=364 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f4876ff5640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 info.flags=0x17 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f4876ff5640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 to cache LRU end 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f4876ff5640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f4876ff5640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f4876ff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f4876ff5640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-43 : miss 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.644+0000 7f4876ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f4876ff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f4876ff5640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-43 info.flags=0x0 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f4876ff5640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-43 to cache LRU end 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f4876ff5640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-43 : miss 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f4876ff5640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f4876ff5640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f4876ff5640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-43 info.flags=0x0 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f4876ff5640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-43 to cache LRU end 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f4876ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding source pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f4876ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3): adding dest pipe: {s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f4876ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f4876ff5640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f4876ff5640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4829fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4829fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4829fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4829fa0:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cab160:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.642 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.643 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.643 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.643 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.643 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.643 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.643 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.643 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.643 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 2d76611e372ca6154bc7865d7ac3ba413f7ba7bf4ab9ed01156f09f41b593ccb 2026-03-20T11:47:14.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.643 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.643 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.643 INFO:teuthology.orchestra.run.vm02.stderr:2d76611e372ca6154bc7865d7ac3ba413f7ba7bf4ab9ed01156f09f41b593ccb 2026-03-20T11:47:14.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 4c357387f9d0a90f42affa8ddb46c213955a5612a34ac7521b96596167b07c27 2026-03-20T11:47:14.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=4c357387f9d0a90f42affa8ddb46c213955a5612a34ac7521b96596167b07c27 2026-03-20T11:47:14.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-43%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=44, curl_handle=0x55fdc4cbda00 2026-03-20T11:47:14.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f48c3739900 20 run: stack=0x55fdc4cf64d0 is io blocked 2026-03-20T11:47:14.643 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.645+0000 7f481e7fc640 20 link_request req_data=0x55fdc48e75f0 req_data->id=44, curl_handle=0x55fdc4cbda00 2026-03-20T11:47:14.643 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:47:14.646+0000 7f12030f6640 -1 req 2961588121294927649 0.001000001s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:47:14.647 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.651+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4831640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.651+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4831640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.651+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4831640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.651+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4831640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.651+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc482c640:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.651+0000 7f48c3739900 15 stack 0x55fdc4d01f90 end 2026-03-20T11:47:14.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.651+0000 7f48c3739900 20 run: stack=0x55fdc4d01f90 is done 2026-03-20T11:47:14.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.651+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.651+0000 7f48c3739900 20 collect(): s=0x55fdc4bb3e20 stack=0x55fdc4d01f90 is complete 2026-03-20T11:47:14.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.651+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.651+0000 7f48c3739900 10 trimming bilogs for bucket=:dipepu-39[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44]) markers=[,,,,,,,,00000000001.18.6,,], shards=11 2026-03-20T11:47:14.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.651+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc482c640:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.651+0000 7f48c3739900 10 trimming bilog shard 8 of :dipepu-39[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44]) at marker 00000000001.18.6 2026-03-20T11:47:14.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.651+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.651+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cb0940:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.651+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cb0940:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.651+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44.8 2026-03-20T11:47:14.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.651+0000 7f48c3739900 20 run: stack=0x55fdc4d01f90 is io blocked 2026-03-20T11:47:14.648 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cb0940:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cb0940:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48c3739900 15 stack 0x55fdc4d01f90 end 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48c3739900 20 run: stack=0x55fdc4d01f90 is done 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc482c640:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48c3739900 20 trim: trimmed bucket instance dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48c3739900 15 stack 0x55fdc4bb3e20 end 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is done 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48c3739900 4 starting trim on bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cb1190:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cb1190:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48c3739900 20 enqueued request req=0x55fdc4cf6880 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48c3739900 20 req: 0x55fdc4cf6880 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is io blocked 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48767f4640 20 dequeued request req=0x55fdc4cf6880 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48767f4640 20 RGWWQ: empty 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48767f4640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 : miss 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48767f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48767f4640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48767f4640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 info.flags=0x17 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48767f4640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 to cache LRU end 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48767f4640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48767f4640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48767f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48767f4640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-10 : miss 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48767f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48767f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48767f4640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-10 info.flags=0x0 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48767f4640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-10 to cache LRU end 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48767f4640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-10 : miss 2026-03-20T11:47:14.649 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.652+0000 7f48767f4640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48767f4640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48767f4640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-10 info.flags=0x0 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48767f4640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-10 to cache LRU end 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48767f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding source pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48767f4640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15): adding dest pipe: {s={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48767f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48767f4640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48767f4640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cb1190:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cb1190:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cb1190:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cb1190:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc482c640:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4831640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4831640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 3a53e398a78e643914ef1f8934c8c9f25c663a66def29bdd432a507526c53d09 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:3a53e398a78e643914ef1f8934c8c9f25c663a66def29bdd432a507526c53d09 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = f9fb4a205b0bc6063c354b3c62dcff4f63644606b8822aeda5908535d7e537ec 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=f9fb4a205b0bc6063c354b3c62dcff4f63644606b8822aeda5908535d7e537ec 2026-03-20T11:47:14.650 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.651 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.651 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-10%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.651 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=45, curl_handle=0x55fdc48e6af0 2026-03-20T11:47:14.651 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f48c3739900 20 run: stack=0x55fdc4d01f90 is io blocked 2026-03-20T11:47:14.651 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.653+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cbe150 req_data->id=45, curl_handle=0x55fdc48e6af0 2026-03-20T11:47:14.651 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4cb1f80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4cb1f80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4cb1f80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4cb1f80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4cfaee0:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 15 stack 0x55fdc4cf5990 end 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 run: stack=0x55fdc4cf5990 is done 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 collect(): s=0x55fdc48e4c10 stack=0x55fdc4cf5990 is complete 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 10 trimming bilogs for bucket=:dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) markers=[,00000000002.13.6,,00000000001.5.6,,00000000001.2.6,00000000002.13.6,00000000001.11.6,00000000001.4.6,00000000001.12.6,], shards=11 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc482ddf0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 10 trimming bilog shard 1 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000002.13.6 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 10 trimming bilog shard 3 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.5.6 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 10 trimming bilog shard 5 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.2.6 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 10 trimming bilog shard 6 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000002.13.6 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 10 trimming bilog shard 7 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.11.6 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 10 trimming bilog shard 8 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.4.6 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 10 trimming bilog shard 9 of :dipepu-8[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13]) at marker 00000000001.12.6 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4cf9510:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cfdf40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cb0820:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cb6960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4d1da30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 cr:s=0x55fdc4cc7a10:op=0x55fdc4cb4da0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 cr:s=0x55fdc482f260:op=0x55fdc4cfaee0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4cf9510:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.1 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 run: stack=0x55fdc4cf5990 is io blocked 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cfdf40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.3 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 run: stack=0x55fdc4ca7c70 is io blocked 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cb0820:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.5 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is io blocked 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cb6960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.6 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is io blocked 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4d1da30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.7 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 run: stack=0x55fdc4d025f0 is io blocked 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 cr:s=0x55fdc4cc7a10:op=0x55fdc4cb4da0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.8 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 run: stack=0x55fdc4cc7a10 is io blocked 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 cr:s=0x55fdc482f260:op=0x55fdc4cfaee0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13.9 2026-03-20T11:47:14.652 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.655+0000 7f48c3739900 20 run: stack=0x55fdc482f260 is io blocked 2026-03-20T11:47:14.653 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 cr:s=0x55fdc4cc7a10:op=0x55fdc4cb4da0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.653 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 cr:s=0x55fdc4cc7a10:op=0x55fdc4cb4da0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.653 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 15 stack 0x55fdc4cc7a10 end 2026-03-20T11:47:14.653 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.653 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 run: stack=0x55fdc4cc7a10 is done 2026-03-20T11:47:14.653 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc482ddf0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.653 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.653 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 cr:s=0x55fdc482f260:op=0x55fdc4cfaee0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.653 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 cr:s=0x55fdc482f260:op=0x55fdc4cfaee0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.653 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 15 stack 0x55fdc482f260 end 2026-03-20T11:47:14.653 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.653 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 run: stack=0x55fdc482f260 is done 2026-03-20T11:47:14.653 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc482ddf0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.653 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.653 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4cf9510:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.653 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4cf9510:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.653 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 15 stack 0x55fdc4cf5990 end 2026-03-20T11:47:14.653 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.653 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 run: stack=0x55fdc4cf5990 is done 2026-03-20T11:47:14.653 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc482ddf0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.653 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.653 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cb6960:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cb6960:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 15 stack 0x55fdc4cf32a0 end 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is done 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc482ddf0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4d1da30:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 cr:s=0x55fdc4d025f0:op=0x55fdc4d1da30:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 15 stack 0x55fdc4d025f0 end 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 run: stack=0x55fdc4d025f0 is done 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc482ddf0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cfdf40:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 cr:s=0x55fdc4ca7c70:op=0x55fdc4cfdf40:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 15 stack 0x55fdc4ca7c70 end 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 run: stack=0x55fdc4ca7c70 is done 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc482ddf0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.656+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4cf6b90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4cf6b90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cb0820:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 cr:s=0x55fdc4cbbce0:op=0x55fdc4cb0820:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 15 stack 0x55fdc4cbbce0 end 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 run: stack=0x55fdc4cbbce0 is done 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4cf6b90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc482ddf0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4cf6b90:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 cr:s=0x55fdc48e4c10:op=0x55fdc4d08b80:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 trim: trimmed bucket instance dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 15 stack 0x55fdc48e4c10 end 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 run: stack=0x55fdc48e4c10 is done 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4831d30:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 15 stack 0x55fdc4ca6700 end 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 run: stack=0x55fdc4ca6700 is done 2026-03-20T11:47:14.654 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 collect(): s=0x55fdc4bc6f60 stack=0x55fdc4ca6700 is complete 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 10 trimming bilogs for bucket=:dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) markers=[,,,,,00000000002.22.11,,,00000000031.119.12,,], shards=11 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4d08990:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 4 starting trim on bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc482ddf0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 10 trimming bilog shard 5 of :dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) at marker 00000000002.22.11 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 10 trimming bilog shard 8 of :dipepu-16[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21]) at marker 00000000031.119.12 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cb1190:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4cb0820:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4cf15a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cb1190:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 enqueued request req=0x55fdc4d062a0 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 req: 0x55fdc4d062a0 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is io blocked 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4cb0820:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21.5 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 run: stack=0x55fdc4ca6700 is io blocked 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4cf15a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21.8 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f48c3739900 20 run: stack=0x55fdc4cf5990 is io blocked 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f4875ff3640 20 dequeued request req=0x55fdc4d062a0 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f4875ff3640 20 RGWWQ: empty 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f4875ff3640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 : miss 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f4875ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f4875ff3640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=363 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f4875ff3640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 info.flags=0x17 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f4875ff3640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 to cache LRU end 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f4875ff3640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f4875ff3640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f4875ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f4875ff3640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-3 : miss 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f4875ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f4875ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f4875ff3640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-3 info.flags=0x0 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f4875ff3640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-3 to cache LRU end 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f4875ff3640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-3 : miss 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f4875ff3640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.655 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f4875ff3640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f4875ff3640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-3 info.flags=0x0 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f4875ff3640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-3 to cache LRU end 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f4875ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding source pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.657+0000 7f4875ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4): adding dest pipe: {s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f4875ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f4875ff3640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f4875ff3640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cb1190:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cb1190:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cb1190:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cb1190:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4d08990:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 cr:s=0x55fdc482f260:op=0x55fdc4d01100:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4d08990:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 cr:s=0x55fdc482f260:op=0x55fdc4d1e340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 cr:s=0x55fdc482f260:op=0x55fdc4d1e340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = c4890dc79158e09d2de2a46ded29acb8e08789f41986d9cbe56bd417b6a0c6c7 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:c4890dc79158e09d2de2a46ded29acb8e08789f41986d9cbe56bd417b6a0c6c7 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 0b84d2b8942a536d0af8379c0891b5957ba6b17386d460e0070299cc08833859 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=0b84d2b8942a536d0af8379c0891b5957ba6b17386d460e0070299cc08833859 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-3%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=46, curl_handle=0x55fdc4cb95e0 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 run: stack=0x55fdc482f260 is io blocked 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cb9d30 req_data->id=46, curl_handle=0x55fdc4cb95e0 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4cf15a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 cr:s=0x55fdc4cf5990:op=0x55fdc4cf15a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 15 stack 0x55fdc4cf5990 end 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 run: stack=0x55fdc4cf5990 is done 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc482ddf0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.656 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4cb0820:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4cb0820:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 15 stack 0x55fdc4ca6700 end 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 run: stack=0x55fdc4ca6700 is done 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc482ddf0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 trim: trimmed bucket instance dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 15 stack 0x55fdc4bc6f60 end 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is done 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 4 starting trim on bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cb1190:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cb1190:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 enqueued request req=0x55fdc4ca5fd0 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 req: 0x55fdc4ca5fd0 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is io blocked 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48757f2640 20 dequeued request req=0x55fdc4ca5fd0 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48757f2640 20 RGWWQ: empty 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48757f2640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 : miss 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48757f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48757f2640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=365 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48757f2640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 info.flags=0x17 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48757f2640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 to cache LRU end 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48757f2640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48757f2640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48757f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48757f2640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-9 : miss 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48757f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48757f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48757f2640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-9 info.flags=0x0 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48757f2640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-9 to cache LRU end 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48757f2640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-9 : miss 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.658+0000 7f48757f2640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.657 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:47:14.658+0000 7f12751da640 -1 req 5657203836078576233 0.000000000s :get_bucket_index_log_status ERROR: rgw_read_bucket_full_sync_status() on pipe={s={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} returned ret=-2 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48757f2640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48757f2640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-9 info.flags=0x0 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48757f2640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-9 to cache LRU end 2026-03-20T11:47:14.657 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48757f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding source pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48757f2640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14): adding dest pipe: {s={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48757f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48757f2640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48757f2640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cb1190:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cb1190:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cb1190:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4cb1190:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4831d10:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4830a80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4830a80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = f81d189f6f02159fe2468171b75db139ccc1e6705b68fd5c2cf0a2f86a451be3 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:f81d189f6f02159fe2468171b75db139ccc1e6705b68fd5c2cf0a2f86a451be3 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 94cb11d0c2e0af9042e2d1d44fefea43f8ceb5508e09f9da51ca7e43cdf484fe 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=94cb11d0c2e0af9042e2d1d44fefea43f8ceb5508e09f9da51ca7e43cdf484fe 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-9%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=47, curl_handle=0x55fdc4cc1a40 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f48c3739900 20 run: stack=0x55fdc4ca6700 is io blocked 2026-03-20T11:47:14.658 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.659+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cc2150 req_data->id=47, curl_handle=0x55fdc4cc1a40 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cfc150:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cab160:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f48c3739900 15 stack 0x55fdc4cf64d0 end 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f48c3739900 20 run: stack=0x55fdc4cf64d0 is done 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f48c3739900 20 collect(): s=0x55fdc4ca7e70 stack=0x55fdc4cf64d0 is complete 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f48c3739900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f48c3739900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03980:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f48c3739900 15 stack 0x55fdc4ca7e70 end 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f48c3739900 20 stack->operate() returned ret=-22 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is done 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f48c3739900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03810:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f48c3739900 4 starting trim on bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cf8100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cf8100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f48c3739900 20 enqueued request req=0x55fdc4cac740 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f48c3739900 20 req: 0x55fdc4cac740 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is io blocked 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f4874ff1640 20 dequeued request req=0x55fdc4cac740 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f4874ff1640 20 RGWWQ: empty 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f4874ff1640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 : miss 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f4874ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f4874ff1640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f4874ff1640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 info.flags=0x17 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f4874ff1640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 to cache LRU end 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f4874ff1640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f4874ff1640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.684 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f4874ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f4874ff1640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-38 : miss 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.687+0000 7f4874ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f4874ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f4874ff1640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-38 info.flags=0x0 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f4874ff1640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-38 to cache LRU end 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f4874ff1640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-38 : miss 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f4874ff1640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f4874ff1640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f4874ff1640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-38 info.flags=0x0 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f4874ff1640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-38 to cache LRU end 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f4874ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding source pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f4874ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43): adding dest pipe: {s={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f4874ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f4874ff1640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f4874ff1640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cf8100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cf8100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cf8100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4cf8100:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03810:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cfc150:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03810:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cab160:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cab160:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.685 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.686 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.686 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.686 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.686 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.686 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.686 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = 33cc7bcde2a65e1bcc33b7ff84f6296a1ad2799129e0e504512012e3f1ddbc26 2026-03-20T11:47:14.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.686 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.686 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.686 INFO:teuthology.orchestra.run.vm02.stderr:33cc7bcde2a65e1bcc33b7ff84f6296a1ad2799129e0e504512012e3f1ddbc26 2026-03-20T11:47:14.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = d8e81b1e8525902bb0e405a4dbf7df96fc11a36ff96eb50f5475e34dc2450074 2026-03-20T11:47:14.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=d8e81b1e8525902bb0e405a4dbf7df96fc11a36ff96eb50f5475e34dc2450074 2026-03-20T11:47:14.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-38%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=48, curl_handle=0x55fdc4cbda00 2026-03-20T11:47:14.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f48c3739900 20 run: stack=0x55fdc4cf64d0 is io blocked 2026-03-20T11:47:14.686 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.688+0000 7f481e7fc640 20 link_request req_data=0x55fdc48e75f0 req_data->id=48, curl_handle=0x55fdc4cbda00 2026-03-20T11:47:14.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.695+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4831640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.695+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4831640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.695+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4831640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.695+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4831640:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.695+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc482c640:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.695+0000 7f48c3739900 15 stack 0x55fdc4d01f90 end 2026-03-20T11:47:14.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.695+0000 7f48c3739900 20 run: stack=0x55fdc4d01f90 is done 2026-03-20T11:47:14.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.695+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.695+0000 7f48c3739900 20 collect(): s=0x55fdc4bb3e20 stack=0x55fdc4d01f90 is complete 2026-03-20T11:47:14.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.695+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.695+0000 7f48c3739900 10 trimming bilogs for bucket=:dipepu-10[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15]) markers=[,00000000002.9.6,,,,,,,,,], shards=11 2026-03-20T11:47:14.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.695+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc482c640:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.695+0000 7f48c3739900 10 trimming bilog shard 1 of :dipepu-10[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15]) at marker 00000000002.9.6 2026-03-20T11:47:14.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.695+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.695+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cfcfd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.695+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cfcfd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.695+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15.1 2026-03-20T11:47:14.692 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.695+0000 7f48c3739900 20 run: stack=0x55fdc4d01f90 is io blocked 2026-03-20T11:47:14.693 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cfcfd0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cfcfd0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48c3739900 15 stack 0x55fdc4d01f90 end 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48c3739900 20 run: stack=0x55fdc4d01f90 is done 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc482c640:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48c3739900 20 trim: trimmed bucket instance dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48c3739900 15 stack 0x55fdc4bb3e20 end 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is done 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48c3739900 4 starting trim on bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cfe060:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cfe060:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48c3739900 20 enqueued request req=0x55fdc4cf6880 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48c3739900 20 RGWWQ: 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48c3739900 20 req: 0x55fdc4cf6880 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is io blocked 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48747f0640 20 dequeued request req=0x55fdc4cf6880 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48747f0640 20 RGWWQ: empty 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48747f0640 10 rgw async rados processor: cache get: name=a2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 : miss 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48747f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48747f0640 20 rgw async rados processor: rados_obj.operate() r=0 bl.length=366 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48747f0640 10 rgw async rados processor: cache put: name=a2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 info.flags=0x17 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48747f0640 10 rgw async rados processor: adding a2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 to cache LRU end 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48747f0640 10 rgw async rados processor: updating xattr: name=ceph.objclass.version bl.length()=42 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48747f0640 10 rgw async rados processor: updating xattr: name=user.rgw.acl bl.length()=185 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48747f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48747f0640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-source-hints.dipepu-15 : miss 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48747f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48747f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48747f0640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-source-hints.dipepu-15 info.flags=0x0 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48747f0640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-source-hints.dipepu-15 to cache LRU end 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48747f0640 10 rgw async rados processor: cache get: name=a2.rgw.log++bucket.sync-target-hints.dipepu-15 : miss 2026-03-20T11:47:14.694 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.697+0000 7f48747f0640 20 rgw async rados processor: rados->read ofs=0 len=0 2026-03-20T11:47:14.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48747f0640 20 rgw async rados processor: rados_obj.operate() r=-2 bl.length=0 2026-03-20T11:47:14.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48747f0640 10 rgw async rados processor: cache put: name=a2.rgw.log++bucket.sync-target-hints.dipepu-15 info.flags=0x0 2026-03-20T11:47:14.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48747f0640 10 rgw async rados processor: adding a2.rgw.log++bucket.sync-target-hints.dipepu-15 to cache LRU end 2026-03-20T11:47:14.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48747f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding source pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48747f0640 20 rgw async rados processor: reflect(): flow manager (bucket=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20): adding dest pipe: {s={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48747f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding source pipe: {s={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0},d={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0}} 2026-03-20T11:47:14.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48747f0640 20 rgw async rados processor: reflect(): flow manager (bucket=): adding dest pipe: {s={b=*,z=51055576-6662-409d-8abb-ecaeefd3dd80,az=0},d={b=*,z=4b634ebf-58f5-45bc-839a-cb4b425e773a,az=0}} 2026-03-20T11:47:14.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48747f0640 10 rgw async rados processor: chain_cache_entry: cache_locator=a2.rgw.meta+root+.bucket.meta.dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:47:14.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cfe060:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cfe060:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cfe060:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cfe060:16RGWSimpleAsyncCRI33rgw_bucket_get_sync_policy_params33rgw_bucket_get_sync_policy_resultE: operate() 2026-03-20T11:47:14.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc482c640:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cf6be0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cf6be0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 20 get_url picked endpoint=http://vm02.local:8000 2026-03-20T11:47:14.695 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_DATE -> Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4():> HTTP_HOST -> vm02.local:8000 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical headers format = date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]payload request hash = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request = GET 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:/admin/log/ 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&options=merge&rgwx-zonegroup=a&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&status=&type=bucket-index&version=2 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:date:Fri, 20 Mar 2026 11:47:14 +0000 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:host:vm02.local:8000 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:x-amz-date:20260320T114714Z 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr: 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:date;host;x-amz-content-sha256;x-amz-date 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]canonical request hash = c222558d9c8a622442e713a54557ac5c0567c50798c20dc32f1c60cc8522bcf0 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]string to sign = AWS4-HMAC-SHA256 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:20260320T114714Z 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:20260320/a/s3/aws4_request 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:c222558d9c8a622442e713a54557ac5c0567c50798c20dc32f1c60cc8522bcf0 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]date_k = 638c7bc67e4b75e9c407a62660a4150ac0a43f3ee07334b3a5e2a55e7e3c61d9 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]region_k = c10ebd7f80243b1bdb2c330d4d516c57f5a3f5f58b0c9905fe0c1da453c8d5ba 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]service_k = f25d4205fa2f3a80b4ebe3571a673d87e90582e7db0385bb45991b22577b8d2a 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]signing_k = 559de6a7b5f10d69f2c01019c1a84c651e35c5bd468e35bb961f6d1e06fc7153 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 10 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]generated signature = 8bac10d003a5a7048ec02c0de87b3bb225054844d52335d05e51915e88c3f862 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: Authorization: AWS4-HMAC-SHA256 Credential=GRL33KL1ULN5795C/20260320/a/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=8bac10d003a5a7048ec02c0de87b3bb225054844d52335d05e51915e88c3f862 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 20 http_client[GET/http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a]sign_request_v4(): sigv4 header: x-amz-date: 20260320T114714Z 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 20 sending request to http://vm02.local:8000/admin/log/?type=bucket-index&status&options=merge&bucket=dipepu-15%3A4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20&source-zone=51055576-6662-409d-8abb-ecaeefd3dd80&version=2&rgwx-zonegroup=a 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 20 register_request mgr=0x7fff7f35c010 req_data->id=49, curl_handle=0x55fdc48e6af0 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f48c3739900 20 run: stack=0x55fdc4d01f90 is io blocked 2026-03-20T11:47:14.696 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.698+0000 7f481e7fc640 20 link_request req_data=0x55fdc4cbe150 req_data->id=49, curl_handle=0x55fdc48e6af0 2026-03-20T11:47:14.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.700+0000 7f48c3739900 20 cr:s=0x55fdc482f260:op=0x55fdc4d1e340:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.700+0000 7f48c3739900 20 cr:s=0x55fdc482f260:op=0x55fdc4d1e340:21RGWReadRESTResourceCRI12StatusShardsE: operate() returned r=-2 2026-03-20T11:47:14.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.700+0000 7f48c3739900 20 cr:s=0x55fdc482f260:op=0x55fdc4d01100:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.700+0000 7f48c3739900 15 stack 0x55fdc482f260 end 2026-03-20T11:47:14.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.700+0000 7f48c3739900 20 run: stack=0x55fdc482f260 is done 2026-03-20T11:47:14.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.700+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4d08990:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.700+0000 7f48c3739900 20 collect(): s=0x55fdc4cf32a0 stack=0x55fdc482f260 is complete 2026-03-20T11:47:14.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.700+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.700+0000 7f48c3739900 5 int {anonymous}::take_min_status(ceph::common::CephContext*, uint64_t, std::vector::const_iterator, std::vector::const_iterator, std::vector >*, const DoutPrefixProvider*):ERROR: shards don't match. peer shard:0 my shards:11for generation:0 2026-03-20T11:47:14.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.700+0000 7f48c3739900 4 failed to correlate bucket sync status from peers 2026-03-20T11:47:14.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.700+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4d08990:20BucketTrimInstanceCR: operate() returned r=-22 2026-03-20T11:47:14.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.700+0000 7f48c3739900 15 stack 0x55fdc4cf32a0 end 2026-03-20T11:47:14.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.700+0000 7f48c3739900 20 stack->operate() returned ret=-22 2026-03-20T11:47:14.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.700+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is done 2026-03-20T11:47:14.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.700+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.700+0000 7f48c3739900 4 trim: failed to trim bucket instance: (22) Invalid argument 2026-03-20T11:47:14.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.700+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.697 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.701+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4830a80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.698 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.701+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4830a80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.698 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.701+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4830a80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.698 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.701+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4830a80:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.698 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.701+0000 7f48c3739900 20 cr:s=0x55fdc4ca6700:op=0x55fdc4831d10:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.698 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.701+0000 7f48c3739900 15 stack 0x55fdc4ca6700 end 2026-03-20T11:47:14.698 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.701+0000 7f48c3739900 20 run: stack=0x55fdc4ca6700 is done 2026-03-20T11:47:14.698 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.701+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.698 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.701+0000 7f48c3739900 20 collect(): s=0x55fdc4bc6f60 stack=0x55fdc4ca6700 is complete 2026-03-20T11:47:14.698 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.701+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.698 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.701+0000 7f48c3739900 10 trimming bilogs for bucket=:dipepu-9[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14]) markers=[,00000000002.17.6,,,,,,,,,], shards=11 2026-03-20T11:47:14.698 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.701+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4d081c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.698 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.701+0000 7f48c3739900 10 trimming bilog shard 1 of :dipepu-9[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14]) at marker 00000000002.17.6 2026-03-20T11:47:14.698 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.701+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.698 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.701+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cf15a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.698 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.701+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cf15a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.698 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.701+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14.1 2026-03-20T11:47:14.698 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.701+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is io blocked 2026-03-20T11:47:14.699 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.703+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cf15a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.699 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.703+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cf15a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.699 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.703+0000 7f48c3739900 15 stack 0x55fdc4cf32a0 end 2026-03-20T11:47:14.699 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.703+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.699 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.703+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is done 2026-03-20T11:47:14.699 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.703+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4d081c0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.699 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.703+0000 7f48c3739900 20 cr:s=0x55fdc4bc6f60:op=0x55fdc4ca4bd0:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.700 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.703+0000 7f48c3739900 20 trim: trimmed bucket instance dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14 2026-03-20T11:47:14.700 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.703+0000 7f48c3739900 15 stack 0x55fdc4bc6f60 end 2026-03-20T11:47:14.700 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.703+0000 7f48c3739900 20 run: stack=0x55fdc4bc6f60 is done 2026-03-20T11:47:14.700 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.703+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.700 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.703+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.731+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cab160:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.731+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cab160:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.731+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cab160:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.731+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cab160:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.731+0000 7f48c3739900 20 cr:s=0x55fdc4cf64d0:op=0x55fdc4cfc150:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.731+0000 7f48c3739900 15 stack 0x55fdc4cf64d0 end 2026-03-20T11:47:14.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.731+0000 7f48c3739900 20 run: stack=0x55fdc4cf64d0 is done 2026-03-20T11:47:14.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.731+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03810:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.731+0000 7f48c3739900 20 collect(): s=0x55fdc4ca7e70 stack=0x55fdc4cf64d0 is complete 2026-03-20T11:47:14.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.731+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.731+0000 7f48c3739900 10 trimming bilogs for bucket=:dipepu-38[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43]) markers=[,,,,,,,,00000000001.55.6,,], shards=11 2026-03-20T11:47:14.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.731+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4829fa0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.731+0000 7f48c3739900 10 trimming bilog shard 8 of :dipepu-38[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43]) at marker 00000000001.55.6 2026-03-20T11:47:14.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.731+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.731+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4d064e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.731+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4d064e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.731+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43.8 2026-03-20T11:47:14.728 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.731+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is io blocked 2026-03-20T11:47:14.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.732+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4d064e0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.732+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4d064e0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.732+0000 7f48c3739900 15 stack 0x55fdc4cf32a0 end 2026-03-20T11:47:14.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.732+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.732+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is done 2026-03-20T11:47:14.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.732+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4829fa0:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.732+0000 7f48c3739900 20 cr:s=0x55fdc4ca7e70:op=0x55fdc4d03810:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.732+0000 7f48c3739900 20 trim: trimmed bucket instance dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43 2026-03-20T11:47:14.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.732+0000 7f48c3739900 15 stack 0x55fdc4ca7e70 end 2026-03-20T11:47:14.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.732+0000 7f48c3739900 20 run: stack=0x55fdc4ca7e70 is done 2026-03-20T11:47:14.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.732+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.729 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.732+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.740+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cf6be0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.740+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cf6be0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.740+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cf6be0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.740+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc4cf6be0:21RGWReadRESTResourceCRI12StatusShardsE: operate() 2026-03-20T11:47:14.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.740+0000 7f48c3739900 20 cr:s=0x55fdc4d01f90:op=0x55fdc482c640:27RGWReadRemoteStatusShardsCR: operate() 2026-03-20T11:47:14.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.740+0000 7f48c3739900 15 stack 0x55fdc4d01f90 end 2026-03-20T11:47:14.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.740+0000 7f48c3739900 20 run: stack=0x55fdc4d01f90 is done 2026-03-20T11:47:14.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.740+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.740+0000 7f48c3739900 20 collect(): s=0x55fdc4bb3e20 stack=0x55fdc4d01f90 is complete 2026-03-20T11:47:14.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.740+0000 7f48c3739900 10 min_generation is 0 2026-03-20T11:47:14.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.740+0000 7f48c3739900 10 trimming bilogs for bucket=:dipepu-15[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20]) markers=[,,,,,,,,00000000007.15.6,,], shards=11 2026-03-20T11:47:14.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.740+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cb1250:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.740+0000 7f48c3739900 10 trimming bilog shard 8 of :dipepu-15[4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20]) at marker 00000000007.15.6 2026-03-20T11:47:14.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.740+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1 2026-03-20T11:47:14.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.740+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cf15a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.740+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cf15a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.740+0000 7f48c3739900 20 bucket index object: :.dir.4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20.8 2026-03-20T11:47:14.737 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.740+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is io blocked 2026-03-20T11:47:14.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.741+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cf15a0:19RGWRadosBILogTrimCR: operate() 2026-03-20T11:47:14.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.741+0000 7f48c3739900 20 cr:s=0x55fdc4cf32a0:op=0x55fdc4cf15a0:19RGWRadosBILogTrimCR: operate() returned r=-61 2026-03-20T11:47:14.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.741+0000 7f48c3739900 15 stack 0x55fdc4cf32a0 end 2026-03-20T11:47:14.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.741+0000 7f48c3739900 20 stack->operate() returned ret=-61 2026-03-20T11:47:14.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.741+0000 7f48c3739900 20 run: stack=0x55fdc4cf32a0 is done 2026-03-20T11:47:14.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.741+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4cb1250:24BucketTrimShardCollectCR: operate() 2026-03-20T11:47:14.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.741+0000 7f48c3739900 20 cr:s=0x55fdc4bb3e20:op=0x55fdc4ca8780:20BucketTrimInstanceCR: operate() 2026-03-20T11:47:14.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.741+0000 7f48c3739900 20 trim: trimmed bucket instance dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20 2026-03-20T11:47:14.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.741+0000 7f48c3739900 15 stack 0x55fdc4bb3e20 end 2026-03-20T11:47:14.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.741+0000 7f48c3739900 20 run: stack=0x55fdc4bb3e20 is done 2026-03-20T11:47:14.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.741+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() 2026-03-20T11:47:14.738 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.741+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca6f40:27BucketTrimInstanceCollectCR: operate() returned r=-22 2026-03-20T11:47:14.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.741+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca3130:12BucketTrimCR: operate() 2026-03-20T11:47:14.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.742+0000 7f48c3739900 20 writing bucket trim marker=16:2f87aa92:root::.bucket.meta.dipepu-13%3a4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18:head 2026-03-20T11:47:14.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.742+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4cf8100:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:14.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.742+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4cf8100:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:14.739 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.742+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is io blocked 2026-03-20T11:47:14.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.744+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4cf8100:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:14.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.744+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4cf8100:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:14.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.744+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4cf8100:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:14.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.744+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4cf8100:21RGWSimpleRadosWriteCRIN3rgw16BucketTrimStatusEE: operate() 2026-03-20T11:47:14.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.744+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca3130:12BucketTrimCR: operate() 2026-03-20T11:47:14.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.744+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4cf8100:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:14.740 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.744+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4cf8100:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:14.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.744+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is io blocked 2026-03-20T11:47:14.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.744+0000 7f4899ffb640 20 trim: bucket trim completed 2026-03-20T11:47:14.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.744+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4cf8100:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:14.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.744+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4cf8100:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:14.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.744+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4cf8100:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:14.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.744+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4cf8100:16RGWRadosNotifyCR: operate() 2026-03-20T11:47:14.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.744+0000 7f48c3739900 20 cr:s=0x55fdc4ca3830:op=0x55fdc4ca3130:12BucketTrimCR: operate() 2026-03-20T11:47:14.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.744+0000 7f48c3739900 4 bucket index log processing completed in 0.703211963s 2026-03-20T11:47:14.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.744+0000 7f48c3739900 15 stack 0x55fdc4ca3830 end 2026-03-20T11:47:14.741 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.744+0000 7f48c3739900 20 run: stack=0x55fdc4ca3830 is done 2026-03-20T11:47:14.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.750+0000 7f48c3739900 20 remove_watcher() i=4 2026-03-20T11:47:14.747 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.750+0000 7f48c3739900 2 removed watcher, disabling cache 2026-03-20T11:47:14.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.751+0000 7f48c3739900 20 remove_watcher() i=1 2026-03-20T11:47:14.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.751+0000 7f48c3739900 20 remove_watcher() i=3 2026-03-20T11:47:14.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.751+0000 7f48c3739900 20 remove_watcher() i=0 2026-03-20T11:47:14.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.751+0000 7f48c3739900 20 remove_watcher() i=5 2026-03-20T11:47:14.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.751+0000 7f48c3739900 20 remove_watcher() i=6 2026-03-20T11:47:14.748 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.751+0000 7f48c3739900 20 remove_watcher() i=2 2026-03-20T11:47:14.749 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:14.752+0000 7f48c3739900 20 remove_watcher() i=7 2026-03-20T11:47:19.906 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'list', 'bucket.instance', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:47:19.906 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'list', 'bucket.instance', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:47:19.906 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata list bucket.instance --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:47:19.944 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:47:19.944 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:47:20.123 INFO:teuthology.orchestra.run.vm02.stdout:["dipepu-55:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.15","dipepu-28:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.33","dipepu-31:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.36","dipepu-39:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.44","dipepu-8:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.13","dipepu-16:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.21","dipepu-43:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.3","dipepu-10:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.15","dipepu-3:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.4","dipepu-9:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.14","dipepu-38:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.43","dipepu-15:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.20","dipepu-13:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.18","dipepu-11:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.16","dipepu-12:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.17","dipepu-40:4b634ebf-58f5-45bc-839a-cb4b425e773a.5539.1","dipepu-18:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.23","dipepu-50:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.10","dipepu-24:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.29","dipepu-56:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.16","dipepu-27:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.32","dipepu-44:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.4","dipepu-17:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.22","dipepu-46:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.6","dipepu-37:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.42","dipepu-1:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.1","dipepu-7:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.12","dipepu-48:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.8","dipepu-59:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.19","dipepu-21:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.26","dipepu-61:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.21","dipepu-41:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.1","dipepu-49:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.9","dipepu-45:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.5","dipepu-33:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.38","dipepu-47:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.7","dipepu-58:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.18","dipepu-30:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.35","dipepu-20:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.25","dipepu-52:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.12","dipepu-42:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.2","dipepu-2:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.2","dipepu-57:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.17","dipepu-29:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.34","dipepu-23:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.28","dipepu-34:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.39","dipepu-54:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.14","dipepu-32:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.37","dipepu-19:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.24","dipepu-4:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.5","dipepu-53:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.13","dipepu-14:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.19","dipepu-36:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.41","dipepu-22:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.27","dipepu-25:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.30","dipepu-35:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.40","dipepu-26:4b634ebf-58f5-45bc-839a-cb4b425e773a.4395.31","dipepu-51:4b634ebf-58f5-45bc-839a-cb4b425e773a.5720.11"] 2026-03-20T11:47:20.123 CRITICAL:rgw_multi.tests:zone contains bucket instance dipepu-61 2026-03-20T11:47:20.123 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_log_trim_after_delete_bucket_secondary_reshard ... FAIL 2026-03-20T11:47:20.237 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:47:20.237 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:47:20.237 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:47:20.274 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:47:20.274 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:47:20.371 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:47:20.383 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774007137.857782_141.1","last_update":"2026-03-20T11:45:37.857782Z"},{"marker":"1_1774007137.912792_174.1","last_update":"2026-03-20T11:45:37.912792Z"},{"marker":"1_1774007240.236955_283.1","last_update":"2026-03-20T11:47:20.236955Z"},{"marker":"1_1774007039.336409_210.1","last_update":"2026-03-20T11:43:59.336409Z"}] 2026-03-20T11:47:20.383 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:47:20.383 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:47:20.383 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:47:20.383 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:47:20.463 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:47:20.463 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:47:20.635 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:20.638+0000 7f911a13b900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:47:20.636 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:20.639+0000 7f911a13b900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:47:20.646 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774007137.857782_141.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:45:37.857782Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774007137.912792_174.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:37.912792Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774007240.236955_283.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:47:20.236955Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774007039.336409_210.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:59.336409Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:47:20Z"} 2026-03-20T11:47:20.647 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:47:20.680 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-62', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:47:20.680 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-62', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:47:20.680 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-62 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:47:20.728 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:47:20.728 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:47:20.817 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:20.820+0000 7f408b326900 1 waiting to reach incremental sync.. 2026-03-20T11:47:25.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:25.821+0000 7f408b326900 1 bucket sync caught up with source: 2026-03-20T11:47:25.818 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, 00000000001.66.6, , , , 00000000001.93.6, , , 00000000001.36.6, , 00000000001.34.6] 2026-03-20T11:47:25.818 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, 00000000001.66.6, , , , 00000000001.93.6, , , 00000000001.36.6, , 00000000001.34.6] 2026-03-20T11:47:25.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:25.821+0000 7f408b326900 0 bucket checkpoint complete 2026-03-20T11:47:25.828 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-62', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:47:25.828 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-62', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:47:25.828 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-62 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:47:25.907 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:47:25.907 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:47:25.999 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:26.002+0000 7f262c010900 1 waiting to reach incremental sync.. 2026-03-20T11:47:29.326 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:47:29.327+0000 7fcfb49b7640 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2026-03-20T11:47:29.326 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:47:29.327+0000 7fcfb19b1640 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2026-03-20T11:47:29.326 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:47:29.328+0000 7fcfb29b3640 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2026-03-20T11:47:31.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:31.003+0000 7f262c010900 1 bucket sync caught up with source: 2026-03-20T11:47:31.000 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, 00000000001.84.6, , , , 00000000001.104.6, , , 00000000001.55.6, , 00000000001.54.6] 2026-03-20T11:47:31.000 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, 00000000001.84.6, , , , 00000000001.104.6, , , 00000000001.55.6, , 00000000001.54.6] 2026-03-20T11:47:31.000 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:31.003+0000 7f262c010900 0 bucket checkpoint complete 2026-03-20T11:47:31.012 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-62 zones={a2, a1} 2026-03-20T11:47:31.020 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:47:31.020 DEBUG:tasks.rgw_multi.zone_rados:o=a, v=null 2026-03-20T11:47:31.020 DEBUG:tasks.rgw_multi.zone_rados:o=b, v=null 2026-03-20T11:47:31.020 DEBUG:tasks.rgw_multi.zone_rados:o=c, v=null 2026-03-20T11:47:31.020 DEBUG:tasks.rgw_multi.zone_rados:o=d, v=null 2026-03-20T11:47:31.024 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:47:31.024 DEBUG:tasks.rgw_multi.zone_rados:o=a, v=null 2026-03-20T11:47:31.024 DEBUG:tasks.rgw_multi.zone_rados:o=b, v=null 2026-03-20T11:47:31.024 DEBUG:tasks.rgw_multi.zone_rados:o=c, v=null 2026-03-20T11:47:31.024 DEBUG:tasks.rgw_multi.zone_rados:o=d, v=null 2026-03-20T11:47:31.024 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2026-03-20T11:47:31.081 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2026-03-20T11:47:31.134 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2026-03-20T11:47:31.190 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2026-03-20T11:47:31.244 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-62 zones={a2, a1} 2026-03-20T11:47:31.245 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'reshard', '--bucket', 'dipepu-62', '--num-shards', '3', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:47:31.245 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'reshard', '--bucket', 'dipepu-62', '--num-shards', '3', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:47:31.245 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket reshard --bucket dipepu-62 --num-shards 3 --yes-i-really-mean-it --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:47:31.284 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:47:31.284 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:47:31.388 INFO:teuthology.orchestra.run.vm02.stdout:tenant: 2026-03-20T11:47:31.389 INFO:teuthology.orchestra.run.vm02.stdout:bucket name: dipepu-62 2026-03-20T11:47:31.389 INFO:teuthology.orchestra.run.vm02.stdout:start time: 2026-03-20T11:47:31.392145+0000 2026-03-20T11:47:31.391 INFO:teuthology.orchestra.run.vm02.stdout:inventory_entries: 4 2026-03-20T11:47:31.391 INFO:teuthology.orchestra.run.vm02.stdout:end time: 2026-03-20T11:47:31.395092+0000 2026-03-20T11:47:31.400 INFO:teuthology.orchestra.run.vm02.stdout:start time: 2026-03-20T11:47:31.403940+0000 2026-03-20T11:47:31.403 INFO:teuthology.orchestra.run.vm02.stdout:inc_entries: 0 2026-03-20T11:47:31.403 INFO:teuthology.orchestra.run.vm02.stdout:end time: 2026-03-20T11:47:31.407250+0000 2026-03-20T11:47:31.455 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'reshard', '--bucket', 'dipepu-62', '--num-shards', '3', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:47:31.456 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'reshard', '--bucket', 'dipepu-62', '--num-shards', '3', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:47:31.456 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket reshard --bucket dipepu-62 --num-shards 3 --yes-i-really-mean-it --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:47:31.540 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:47:31.540 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:47:31.633 INFO:teuthology.orchestra.run.vm02.stdout:tenant: 2026-03-20T11:47:31.633 INFO:teuthology.orchestra.run.vm02.stdout:bucket name: dipepu-62 2026-03-20T11:47:31.633 INFO:teuthology.orchestra.run.vm02.stdout:start time: 2026-03-20T11:47:31.637005+0000 2026-03-20T11:47:31.634 INFO:teuthology.orchestra.run.vm02.stdout:inventory_entries: 4 2026-03-20T11:47:31.634 INFO:teuthology.orchestra.run.vm02.stdout:end time: 2026-03-20T11:47:31.637872+0000 2026-03-20T11:47:31.640 INFO:teuthology.orchestra.run.vm02.stdout:start time: 2026-03-20T11:47:31.644142+0000 2026-03-20T11:47:31.641 INFO:teuthology.orchestra.run.vm02.stdout:inc_entries: 0 2026-03-20T11:47:31.641 INFO:teuthology.orchestra.run.vm02.stdout:end time: 2026-03-20T11:47:31.644857+0000 2026-03-20T11:47:31.687 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-62', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:47:31.687 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-62', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:47:31.687 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-62 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:47:31.730 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:47:31.731 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:47:31.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:31.821+0000 7fbd07b80900 1 waiting to reach latest gen 2, on 0.. 2026-03-20T11:47:36.818 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:36.821+0000 7fbd07b80900 1 waiting to reach latest gen 2, on 1.. 2026-03-20T11:47:41.819 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:41.822+0000 7fbd07b80900 1 waiting to reach latest gen 2, on 1.. 2026-03-20T11:47:46.819 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:46.822+0000 7fbd07b80900 1 waiting to reach latest gen 2, on 1.. 2026-03-20T11:47:51.820 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:51.823+0000 7fbd07b80900 1 waiting to reach latest gen 2, on 1.. 2026-03-20T11:47:56.820 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:47:56.823+0000 7fbd07b80900 1 waiting to reach latest gen 2, on 1.. 2026-03-20T11:48:01.821 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:48:01.824+0000 7fbd07b80900 1 waiting to reach latest gen 2, on 1.. 2026-03-20T11:48:06.821 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:48:06.825+0000 7fbd07b80900 1 waiting to reach latest gen 2, on 1.. 2026-03-20T11:48:11.822 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:48:11.825+0000 7fbd07b80900 1 waiting to reach latest gen 2, on 1.. 2026-03-20T11:48:16.822 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:48:16.825+0000 7fbd07b80900 1 waiting to reach latest gen 2, on 1.. 2026-03-20T11:48:21.823 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:48:21.826+0000 7fbd07b80900 1 waiting to reach latest gen 2, on 1.. 2026-03-20T11:48:26.823 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:48:26.826+0000 7fbd07b80900 1 waiting to reach latest gen 2, on 1.. 2026-03-20T11:48:31.824 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:48:31.827+0000 7fbd07b80900 1 waiting to reach latest gen 2, on 1.. 2026-03-20T11:48:36.825 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:48:36.829+0000 7fbd07b80900 1 waiting for incremental sync to catch up: 2026-03-20T11:48:36.826 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , ] 2026-03-20T11:48:36.826 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000003.110.6, 00000000002.52.6, 00000000002.94.6] 2026-03-20T11:48:41.826 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:48:41.829+0000 7fbd07b80900 1 waiting for incremental sync to catch up: 2026-03-20T11:48:41.827 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , ] 2026-03-20T11:48:41.827 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000003.110.6, 00000000002.52.6, 00000000002.94.6] 2026-03-20T11:48:46.827 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:48:46.831+0000 7fbd07b80900 1 waiting for incremental sync to catch up: 2026-03-20T11:48:46.828 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , ] 2026-03-20T11:48:46.828 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000003.110.6, 00000000002.52.6, 00000000002.94.6] 2026-03-20T11:48:51.829 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:48:51.832+0000 7fbd07b80900 1 waiting for incremental sync to catch up: 2026-03-20T11:48:51.829 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , ] 2026-03-20T11:48:51.829 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000003.110.6, 00000000002.52.6, 00000000002.94.6] 2026-03-20T11:48:56.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:48:56.833+0000 7fbd07b80900 1 waiting for incremental sync to catch up: 2026-03-20T11:48:56.830 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , ] 2026-03-20T11:48:56.830 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000003.110.6, 00000000002.52.6, 00000000002.94.6] 2026-03-20T11:49:01.830 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:49:01.833+0000 7fbd07b80900 1 waiting for incremental sync to catch up: 2026-03-20T11:49:01.880 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , ] 2026-03-20T11:49:01.880 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000003.110.6, 00000000002.52.6, 00000000002.94.6] 2026-03-20T11:49:06.831 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:49:06.834+0000 7fbd07b80900 1 waiting for incremental sync to catch up: 2026-03-20T11:49:06.831 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , ] 2026-03-20T11:49:06.831 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000003.110.6, 00000000002.52.6, 00000000002.94.6] 2026-03-20T11:49:11.832 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:49:11.835+0000 7fbd07b80900 1 waiting for incremental sync to catch up: 2026-03-20T11:49:11.832 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , ] 2026-03-20T11:49:11.832 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000003.110.6, 00000000002.52.6, 00000000002.94.6] 2026-03-20T11:49:16.833 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:49:16.836+0000 7fbd07b80900 1 waiting for incremental sync to catch up: 2026-03-20T11:49:16.834 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , ] 2026-03-20T11:49:16.834 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000003.110.6, 00000000002.52.6, 00000000002.94.6] 2026-03-20T11:49:21.834 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:49:21.837+0000 7fbd07b80900 1 waiting for incremental sync to catch up: 2026-03-20T11:49:21.834 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, , ] 2026-03-20T11:49:21.834 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000003.110.6, 00000000002.52.6, 00000000002.94.6] 2026-03-20T11:49:26.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:49:26.838+0000 7fbd07b80900 1 waiting for incremental sync to catch up: 2026-03-20T11:49:26.835 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000003.110.6, , ] 2026-03-20T11:49:26.835 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000003.110.6, 00000000002.52.6, 00000000002.94.6] 2026-03-20T11:49:27.563 INFO:tasks.rgw.c1.client.0.vm02.stdout:2026-03-20T11:49:27.566+0000 7f11de8ad640 -1 WARNING: The OSD cluster is overloaded and struggling to complete ops. You need more capacity to serve this level of demand. 2026-03-20T11:49:31.835 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:49:31.839+0000 7fbd07b80900 1 waiting for incremental sync to catch up: 2026-03-20T11:49:31.836 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000003.110.6, , 00000000002.94.6] 2026-03-20T11:49:31.836 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000003.110.6, 00000000002.52.6, 00000000002.94.6] 2026-03-20T11:49:36.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:49:36.839+0000 7fbd07b80900 1 bucket sync caught up with source: 2026-03-20T11:49:36.836 INFO:teuthology.orchestra.run.vm02.stderr: local status: [00000000003.110.6, 00000000002.52.6, 00000000002.94.6] 2026-03-20T11:49:36.836 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [00000000003.110.6, 00000000002.52.6, 00000000002.94.6] 2026-03-20T11:49:36.836 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:49:36.839+0000 7fbd07b80900 0 bucket checkpoint complete 2026-03-20T11:49:36.849 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-62', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:49:36.849 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-62', '--source-zone', 'a2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:49:36.849 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-62 --source-zone a2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:49:36.894 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:49:36.895 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:49:37.026 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:49:37.030+0000 7f9d82820900 1 waiting for incremental sync to catch up: 2026-03-20T11:49:37.026 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, 00000000001.84.6, , , 00000000001.98.6, 00000000001.104.6, , , 00000000002.71.6, , 00000000002.69.6] 2026-03-20T11:49:37.027 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, 00000000002.100.6, , , 00000000001.98.6, 00000000001.104.6, , , 00000000002.71.6, , 00000000002.69.6] 2026-03-20T11:49:42.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:49:42.031+0000 7f9d82820900 1 bucket sync caught up with source: 2026-03-20T11:49:42.028 INFO:teuthology.orchestra.run.vm02.stderr: local status: [, 00000000002.100.6, , , 00000000001.98.6, 00000000001.104.6, , , 00000000002.71.6, , 00000000002.69.6] 2026-03-20T11:49:42.028 INFO:teuthology.orchestra.run.vm02.stderr: remote markers: [, 00000000002.100.6, , , 00000000001.98.6, 00000000001.104.6, , , 00000000002.71.6, , 00000000002.69.6] 2026-03-20T11:49:42.028 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:49:42.031+0000 7f9d82820900 0 bucket checkpoint complete 2026-03-20T11:49:42.042 INFO:tasks.rgw_multi.zone_rados:comparing bucket=dipepu-62 zones={a2, a1} 2026-03-20T11:49:42.052 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2026-03-20T11:49:42.052 DEBUG:tasks.rgw_multi.zone_rados:o=a, v=null 2026-03-20T11:49:42.052 DEBUG:tasks.rgw_multi.zone_rados:o=b, v=null 2026-03-20T11:49:42.053 DEBUG:tasks.rgw_multi.zone_rados:o=c, v=null 2026-03-20T11:49:42.053 DEBUG:tasks.rgw_multi.zone_rados:o=d, v=null 2026-03-20T11:49:42.053 DEBUG:tasks.rgw_multi.zone_rados:o=e, v=null 2026-03-20T11:49:42.053 DEBUG:tasks.rgw_multi.zone_rados:o=f, v=null 2026-03-20T11:49:42.053 DEBUG:tasks.rgw_multi.zone_rados:o=g, v=null 2026-03-20T11:49:42.053 DEBUG:tasks.rgw_multi.zone_rados:o=h, v=null 2026-03-20T11:49:42.056 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2026-03-20T11:49:42.056 DEBUG:tasks.rgw_multi.zone_rados:o=a, v=null 2026-03-20T11:49:42.056 DEBUG:tasks.rgw_multi.zone_rados:o=b, v=null 2026-03-20T11:49:42.056 DEBUG:tasks.rgw_multi.zone_rados:o=c, v=null 2026-03-20T11:49:42.056 DEBUG:tasks.rgw_multi.zone_rados:o=d, v=null 2026-03-20T11:49:42.056 DEBUG:tasks.rgw_multi.zone_rados:o=e, v=null 2026-03-20T11:49:42.056 DEBUG:tasks.rgw_multi.zone_rados:o=f, v=null 2026-03-20T11:49:42.056 DEBUG:tasks.rgw_multi.zone_rados:o=g, v=null 2026-03-20T11:49:42.056 DEBUG:tasks.rgw_multi.zone_rados:o=h, v=null 2026-03-20T11:49:42.056 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=a 2026-03-20T11:49:42.114 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=b 2026-03-20T11:49:42.172 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=c 2026-03-20T11:49:42.234 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=d 2026-03-20T11:49:42.297 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=e 2026-03-20T11:49:42.351 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=f 2026-03-20T11:49:42.411 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=g 2026-03-20T11:49:42.473 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=h 2026-03-20T11:49:42.530 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=dipepu-62 zones={a2, a1} 2026-03-20T11:49:42.533 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_reshard_incremental ... ok 2026-03-20T11:49:42.705 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:49:42.705 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'a1', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:49:42.705 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone a1 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:49:42.744 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:49:42.744 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:49:42.772 INFO:tasks.rgw.c2.client.1.vm08.stdout:2026-03-20T11:49:42.774+0000 7f3ddd7a1640 -1 WARNING: The OSD cluster is overloaded and struggling to complete ops. You need more capacity to serve this level of demand. 2026-03-20T11:49:42.829 INFO:teuthology.orchestra.run.vm02.stderr:No --period given, using current period=cb2fcdce-2c1b-429a-8601-a284ee3a2381 2026-03-20T11:49:42.840 INFO:teuthology.orchestra.run.vm02.stdout:[{"marker":"1_1774007137.857782_141.1","last_update":"2026-03-20T11:45:37.857782Z"},{"marker":"1_1774007137.912792_174.1","last_update":"2026-03-20T11:45:37.912792Z"},{"marker":"1_1774007382.705813_291.1","last_update":"2026-03-20T11:49:42.705813Z"},{"marker":"1_1774007039.336409_210.1","last_update":"2026-03-20T11:43:59.336409Z"}] 2026-03-20T11:49:42.840 INFO:rgw_multi.tests:starting meta checkpoint for zone=a2 2026-03-20T11:49:42.840 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:49:42.840 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2026-03-20T11:49:42.840 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2026-03-20T11:49:42.923 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:49:42.923 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:49:43.023 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:49:43.026+0000 7fba30d24900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:49:43.024 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:49:43.027+0000 7fba30d24900 20 RGW-SYNC:meta: read sync status 2026-03-20T11:49:43.034 INFO:teuthology.orchestra.run.vm02.stdout:{"sync_status":{"info":{"status":"sync","num_shards":4,"period":"cb2fcdce-2c1b-429a-8601-a284ee3a2381","realm_epoch":2},"markers":[{"key":0,"val":{"state":1,"marker":"1_1774007137.857782_141.1","next_step_marker":"","total_entries":1,"pos":0,"timestamp":"2026-03-20T11:45:37.857782Z","realm_epoch":2}},{"key":1,"val":{"state":1,"marker":"1_1774007137.912792_174.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:45:37.912792Z","realm_epoch":2}},{"key":2,"val":{"state":1,"marker":"1_1774007382.705813_291.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:49:42.705813Z","realm_epoch":2}},{"key":3,"val":{"state":1,"marker":"1_1774007039.336409_210.1","next_step_marker":"","total_entries":0,"pos":0,"timestamp":"2026-03-20T11:43:59.336409Z","realm_epoch":2}}]},"full_sync":{"total":1,"complete":1},"current_time":"2026-03-20T11:49:43Z"} 2026-03-20T11:49:43.034 INFO:rgw_multi.tests:finish meta checkpoint for zone=a2 2026-03-20T11:49:43.034 DEBUG:tasks.rgw.c1.client.1:waiting for process to exit 2026-03-20T11:49:43.034 INFO:teuthology.orchestra.run:waiting for 300 2026-03-20T11:49:43.035 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:49:43.038+0000 7fd0c83de640 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper term radosgw --rgw-frontends beast port=8001 -n client.1 --cluster c1 -k /etc/ceph/c1.client.1.keyring --log-file /var/log/ceph/rgw.c1.client.1.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c1.client.1.sock --foreground --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm (PID: 57604) UID: 0 2026-03-20T11:49:43.035 INFO:tasks.rgw.c1.client.1.vm05.stdout:2026-03-20T11:49:43.038+0000 7fd0cbe80980 -1 shutting down 2026-03-20T11:50:07.439 INFO:tasks.rgw.c1.client.1:Stopped 2026-03-20T11:50:07.479 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'reshard', '--bucket', 'dipepu-63', '--num-shards', '3', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:50:07.479 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'reshard', '--bucket', 'dipepu-63', '--num-shards', '3', '--yes-i-really-mean-it', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2026-03-20T11:50:07.479 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket reshard --bucket dipepu-63 --num-shards 3 --yes-i-really-mean-it --cluster c1 --debug-rgw 0 --debug-ms 0 2026-03-20T11:50:07.519 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:50:07.519 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:50:07.637 INFO:teuthology.orchestra.run.vm02.stdout:tenant: 2026-03-20T11:50:07.637 INFO:teuthology.orchestra.run.vm02.stdout:bucket name: dipepu-63 2026-03-20T11:50:07.638 INFO:teuthology.orchestra.run.vm02.stdout:start time: 2026-03-20T11:50:07.641290+0000 2026-03-20T11:50:07.641 INFO:teuthology.orchestra.run.vm02.stdout:inventory_entries: 4 2026-03-20T11:50:07.641 INFO:teuthology.orchestra.run.vm02.stdout:end time: 2026-03-20T11:50:07.645231+0000 2026-03-20T11:50:07.650 INFO:teuthology.orchestra.run.vm02.stdout:start time: 2026-03-20T11:50:07.654424+0000 2026-03-20T11:50:07.653 INFO:teuthology.orchestra.run.vm02.stdout:inc_entries: 0 2026-03-20T11:50:07.653 INFO:teuthology.orchestra.run.vm02.stdout:end time: 2026-03-20T11:50:07.657378+0000 2026-03-20T11:50:07.731 INFO:tasks.rgw.c1.client.1:Restarting daemon 2026-03-20T11:50:07.731 DEBUG:teuthology.orchestra.run.vm05:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8001' -n client.1 --cluster c1 -k /etc/ceph/c1.client.1.keyring --log-file /var/log/ceph/rgw.c1.client.1.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c1.client.1.sock --foreground --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm | sudo tee /var/log/ceph/rgw.c1.client.1.stdout 2>&1 2026-03-20T11:50:07.733 INFO:tasks.rgw.c1.client.1:Started 2026-03-20T11:50:07.733 DEBUG:teuthology.orchestra.run.vm05:> curl http://vm05.local:8001 2026-03-20T11:50:07.762 INFO:teuthology.orchestra.run.vm05.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2026-03-20T11:50:07.762 INFO:teuthology.orchestra.run.vm05.stderr: Dload Upload Total Spent Left Speed 2026-03-20T11:50:07.762 INFO:teuthology.orchestra.run.vm05.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 2026-03-20T11:50:07.762 INFO:teuthology.orchestra.run.vm05.stderr:curl: (7) Failed to connect to vm05.local port 8001: Connection refused 2026-03-20T11:50:07.762 DEBUG:teuthology.orchestra.run:got remote process result: 7 2026-03-20T11:50:08.763 DEBUG:teuthology.orchestra.run.vm05:> curl http://vm05.local:8001 2026-03-20T11:50:08.786 INFO:teuthology.orchestra.run.vm05.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2026-03-20T11:50:08.786 INFO:teuthology.orchestra.run.vm05.stderr: Dload Upload Total Spent Left Speed 2026-03-20T11:50:08.786 INFO:teuthology.orchestra.run.vm05.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 187 0 187 0 0 182k 0 --:--:-- --:--:-- --:--:-- 182k 2026-03-20T11:50:08.787 INFO:teuthology.orchestra.run.vm05.stdout:anonymous 2026-03-20T11:50:08.787 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-63', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:50:08.787 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'dipepu-63', '--source-zone', 'a1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'a2', '--rgw-zonegroup', 'a', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2026-03-20T11:50:08.787 DEBUG:teuthology.orchestra.run.vm02:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket dipepu-63 --source-zone a1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone a2 --rgw-zonegroup a --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2026-03-20T11:50:08.838 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setuser ceph since I am not root 2026-03-20T11:50:08.838 INFO:teuthology.orchestra.run.vm02.stderr:ignoring --setgroup ceph since I am not root 2026-03-20T11:50:08.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:50:08.938+0000 7f88e2210900 1 waiting to reach incremental sync.. 2026-03-20T11:50:13.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:50:13.938+0000 7f88e2210900 1 waiting to reach incremental sync.. 2026-03-20T11:50:18.935 INFO:teuthology.orchestra.run.vm02.stderr:2026-03-20T11:50:18.939+0000 7f88e2210900 1 waiting to reach incremental sync.. 2026-03-20T11:50:23.446 DEBUG:teuthology.exit:Got signal 15; running 1 handler... 2026-03-20T11:50:23.446 DEBUG:teuthology.exit:Finished running handlers